Update CPU kernels to remove x19

Resolves: COMPMID-5805
Signed-off-by: Michael Tyler <michael.tyler@arm.com>
Change-Id: I250f64531e209625e4ff176dd5a552c1c34bc484
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8909
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gunes Bayir <gunes.bayir@arm.com>
Reviewed-by: Viet-Hoa Do <viet-hoa.do@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

113 files changed, 71740 insertions, 108565 deletions

diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp
index ec93586f57..74791f8d30 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -105,108 +105,108 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 77f\n"
       "beq 39f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 3f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "cbz x14, 4f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 4f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 16f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 15f\n"
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 13f\n"
-      "tbz x13, #3, 8f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 6f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 5f\n"
-      "ldr d16, [x12], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v16.s }[2], [x12]\n"
+      "tbz x14, #3, 8f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 6f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 5f\n"
+      "ldr d16, [x13], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v16.s }[2], [x13]\n"
       "b 12f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s16, [x12, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s16, [x13, #0x0]\n"
       "b 12f\n"
       "6:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x13, #1, 7f\n"
-      "ldr d11, [x12], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v11.s }[2], [x12]\n"
+      "tbz x14, #1, 7f\n"
+      "ldr d11, [x13], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v11.s }[2], [x13]\n"
       "b 12f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s11, [x12, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s11, [x13, #0x0]\n"
       "b 12f\n"
       "8:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x13, #2, 10f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 9f\n"
-      "ldr d10, [x12], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v10.s }[2], [x12]\n"
+      "tbz x14, #2, 10f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 9f\n"
+      "ldr d10, [x13], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v10.s }[2], [x13]\n"
       "b 12f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s10, [x12, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s10, [x13, #0x0]\n"
       "b 12f\n"
       "10:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x13, #1, 11f\n"
-      "ldr d9, [x12], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v9.s }[2], [x12]\n"
+      "tbz x14, #1, 11f\n"
+      "ldr d9, [x13], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v9.s }[2], [x13]\n"
       "b 12f\n"
       "11:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x13, #0x0]\n"
+      "mov x20, #0x0\n"
       "12:"  // Height 1: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 14f\n"
       "13:"  // Height 1: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
       "14:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -227,187 +227,187 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "16:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "17:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 18f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 19f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 19f\n"
       "18:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "19:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 22f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q7, [x11, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q6, [x12, #0x10]\n"
       "blt 21f\n"
       "20:"  // Height 1: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x9, #0x30]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "add x12, x12, #0x40\n"
+      "ldr q7, [x12, #0x0]\n"
       "add x11, x11, #0x40\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "add x10, x10, #0x40\n"
-      "ldr q6, [x11, #0x10]\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "bge 20b\n"
       "21:"  // Height 1: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "sub x26, x26, #0x8\n"
+      "ldr q6, [x9, #0x30]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x12, x12, #0x40\n"
       "add x11, x11, #0x40\n"
       "add x10, x10, #0x40\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "22:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 27f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 27f\n"
+      "cmp x27, #0x4\n"
       "blt 24f\n"
       "23:"  // Height 1: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr q6, [x12, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x4\n"
+      "ldr q7, [x9, #0x10]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
-      "add x28, x28, #0x20\n"
       "bge 23b\n"
       "24:"  // Height 1: Multiply loop: Skip odd blocks
-      "cbz x26, 27f\n"
-      "tbz x26, #1, 25f\n"
-      "ldr s1, [x25], #0x4\n"
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.h }[2], [x25]\n"
+      "cbz x27, 27f\n"
+      "tbz x27, #1, 25f\n"
+      "ldr s1, [x26], #0x4\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.h }[2], [x26]\n"
       "b 26f\n"
       "25:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
       "26:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
-      "add x28, x28, #0x20\n"
       "27:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 17b\n"
       "uzp1 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
       "tbz %x[flags], #1, 28f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -417,189 +417,189 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v10.4s, v10.4s, v0.4s\n"
       "fmax v11.4s, v11.4s, v0.4s\n"
       "28:"  // Height 1: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 37f\n"
-      "tbz x13, #3, 32f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 30f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 29f\n"
-      "str d11, [x12], #0x8\n"
-      "tbz x13, #0, 36f\n"
-      "st1 { v11.s }[2], [x12]\n"
+      "tbz x14, #3, 32f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 30f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 29f\n"
+      "str d11, [x13], #0x8\n"
+      "tbz x14, #0, 36f\n"
+      "st1 { v11.s }[2], [x13]\n"
       "b 36f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 36f\n"
-      "str s11, [x12, #0x0]\n"
+      "tbz x14, #0, 36f\n"
+      "str s11, [x13, #0x0]\n"
       "b 36f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 31f\n"
-      "str d10, [x12], #0x8\n"
-      "tbz x13, #0, 36f\n"
-      "st1 { v10.s }[2], [x12]\n"
+      "tbz x14, #1, 31f\n"
+      "str d10, [x13], #0x8\n"
+      "tbz x14, #0, 36f\n"
+      "st1 { v10.s }[2], [x13]\n"
       "b 36f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 36f\n"
-      "str s10, [x12, #0x0]\n"
+      "tbz x14, #0, 36f\n"
+      "str s10, [x13, #0x0]\n"
       "b 36f\n"
       "32:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 34f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 33f\n"
-      "str d9, [x12], #0x8\n"
-      "tbz x13, #0, 36f\n"
-      "st1 { v9.s }[2], [x12]\n"
+      "tbz x14, #2, 34f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 33f\n"
+      "str d9, [x13], #0x8\n"
+      "tbz x14, #0, 36f\n"
+      "st1 { v9.s }[2], [x13]\n"
       "b 36f\n"
       "33:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 36f\n"
-      "str s9, [x12, #0x0]\n"
+      "tbz x14, #0, 36f\n"
+      "str s9, [x13, #0x0]\n"
       "b 36f\n"
       "34:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 35f\n"
-      "str d8, [x12], #0x8\n"
-      "tbz x13, #0, 36f\n"
-      "st1 { v8.s }[2], [x12]\n"
+      "tbz x14, #1, 35f\n"
+      "str d8, [x13], #0x8\n"
+      "tbz x14, #0, 36f\n"
+      "st1 { v8.s }[2], [x13]\n"
       "b 36f\n"
       "35:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
       "36:"  // Height 1: Partial direct writeback: Done
       "b 38f\n"
       "37:"  // Height 1: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
       "38:"  // Height 1: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 2b\n"
       "b 230f\n"
       "39:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "40:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 41f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 41f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 41f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "41:"  // Height 2: B setup done
-      "cbz x14, 42f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 42f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 54f\n"
       "42:"  // Height 2: no bias
       "tbz %x[flags], #0, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x13, #0x10\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x14, #0x10\n"
+      "add x25, x13, x20, LSL #2\n"
       "bge 51f\n"
-      "tbz x13, #3, 46f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "tbz x13, #2, 44f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 43f\n"
-      "ldr d16, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 50f\n"
-      "ld1 { v16.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "tbz x14, #3, 46f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "tbz x14, #2, 44f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 43f\n"
+      "ldr d16, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 50f\n"
+      "ld1 { v16.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 50f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 50f\n"
-      "ldr s16, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 50f\n"
+      "ldr s16, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 50f\n"
       "44:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x13, #1, 45f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 50f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "tbz x14, #1, 45f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 50f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 50f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 50f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 50f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 50f\n"
       "46:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x13, #2, 48f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 47f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 50f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "tbz x14, #2, 48f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 47f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 50f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 50f\n"
       "47:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 50f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 50f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 50f\n"
       "48:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x13, #1, 49f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 50f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "tbz x14, #1, 49f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 50f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 50f\n"
       "49:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "50:"  // Height 2: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 52f\n"
       "51:"  // Height 2: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "52:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -620,193 +620,193 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "54:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "55:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 56f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 57f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 57f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 57f\n"
       "56:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "57:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 60f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "blt 59f\n"
       "58:"  // Height 2: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "ldr q6, [x9, #0x30]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
-      "ldr q7, [x11, #0x0]\n"
+      "add x12, x12, #0x40\n"
+      "ldr q7, [x12, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
+      "add x11, x11, #0x40\n"
       "add x10, x10, #0x40\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "bge 58b\n"
       "59:"  // Height 2: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "sub x26, x26, #0x8\n"
+      "ldr q6, [x9, #0x30]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
+      "add x12, x12, #0x40\n"
       "add x11, x11, #0x40\n"
       "add x10, x10, #0x40\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "60:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 65f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 65f\n"
+      "cmp x27, #0x4\n"
       "blt 62f\n"
       "61:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x11, #0x10]\n"
+      "sub x27, x27, #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "cmp x26, #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
-      "add x28, x28, #0x20\n"
       "bge 61b\n"
       "62:"  // Height 2: Multiply loop: Skip odd blocks
-      "cbz x26, 65f\n"
-      "tbz x26, #1, 63f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "tbz x26, #0, 64f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
+      "cbz x27, 65f\n"
+      "tbz x27, #1, 63f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "tbz x27, #0, 64f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
       "b 64f\n"
       "63:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
       "64:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
-      "add x28, x28, #0x20\n"
       "65:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 55b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "add x24, x12, x19, LSL #2\n"
+      "add x25, x13, x20, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
@@ -814,10 +814,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "tbz %x[flags], #1, 66f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v7.4s, v7.4s, v1.4s\n"
       "fmin v12.4s, v12.4s, v1.4s\n"
       "fmin v13.4s, v13.4s, v1.4s\n"
@@ -835,120 +835,120 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v10.4s, v10.4s, v0.4s\n"
       "fmax v11.4s, v11.4s, v0.4s\n"
       "66:"  // Height 2: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 75f\n"
-      "tbz x13, #3, 70f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "tbz x13, #2, 68f\n"
-      "st1 { v13.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 67f\n"
-      "str d14, [x12], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "tbz x13, #0, 74f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x24]\n"
+      "tbz x14, #3, 70f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "tbz x14, #2, 68f\n"
+      "st1 { v13.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 67f\n"
+      "str d14, [x13], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "tbz x14, #0, 74f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x25]\n"
       "b 74f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 74f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
+      "tbz x14, #0, 74f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
       "b 74f\n"
       "68:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 69f\n"
-      "str d13, [x12], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "tbz x13, #0, 74f\n"
-      "st1 { v13.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x24]\n"
+      "tbz x14, #1, 69f\n"
+      "str d13, [x13], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "tbz x14, #0, 74f\n"
+      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x25]\n"
       "b 74f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 74f\n"
-      "str s13, [x12, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
+      "tbz x14, #0, 74f\n"
+      "str s13, [x13, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
       "b 74f\n"
       "70:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 72f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 71f\n"
-      "str d12, [x12], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "tbz x13, #0, 74f\n"
-      "st1 { v12.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x24]\n"
+      "tbz x14, #2, 72f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 71f\n"
+      "str d12, [x13], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "tbz x14, #0, 74f\n"
+      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x25]\n"
       "b 74f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 74f\n"
-      "str s12, [x12, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
+      "tbz x14, #0, 74f\n"
+      "str s12, [x13, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
       "b 74f\n"
       "72:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 73f\n"
-      "str d7, [x12], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "tbz x13, #0, 74f\n"
-      "st1 { v7.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x24]\n"
+      "tbz x14, #1, 73f\n"
+      "str d7, [x13], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "tbz x14, #0, 74f\n"
+      "st1 { v7.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x25]\n"
       "b 74f\n"
       "73:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s7, [x12, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
+      "str s7, [x13, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
       "74:"  // Height 2: Partial direct writeback: Done
       "b 76f\n"
       "75:"  // Height 2: Full writeback
-      "str q7, [x12, #0x0]\n"
-      "str q12, [x12, #0x10]\n"
-      "str q13, [x12, #0x20]\n"
-      "str q14, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
+      "str q7, [x13, #0x0]\n"
+      "str q12, [x13, #0x10]\n"
+      "str q13, [x13, #0x20]\n"
+      "str q14, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
       "76:"  // Height 2: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 40b\n"
       "b 230f\n"
       "77:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "78:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 79f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 79f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 79f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "79:"  // Height 3: B setup done
-      "cbz x14, 80f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 80f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "mov v16.16b, v8.16b\n"
@@ -962,111 +962,111 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "b 92f\n"
       "80:"  // Height 3: no bias
       "tbz %x[flags], #0, 91f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 89f\n"
-      "tbz x13, #3, 84f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "tbz x13, #2, 82f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 81f\n"
-      "ldr d16, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x23], #0x8\n"
-      "tbz x13, #0, 88f\n"
-      "ld1 { v16.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
+      "tbz x14, #3, 84f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "tbz x14, #2, 82f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 81f\n"
+      "ldr d16, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "tbz x14, #0, 88f\n"
+      "ld1 { v16.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
       "b 88f\n"
       "81:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 88f\n"
-      "ldr s16, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 88f\n"
+      "ldr s16, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
       "b 88f\n"
       "82:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x13, #1, 83f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "tbz x13, #0, 88f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "tbz x14, #1, 83f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "tbz x14, #0, 88f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 88f\n"
       "83:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 88f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 88f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 88f\n"
       "84:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x13, #2, 86f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 85f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d18, [x23], #0x8\n"
-      "tbz x13, #0, 88f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "tbz x14, #2, 86f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 85f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "tbz x14, #0, 88f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 88f\n"
       "85:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 88f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 88f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 88f\n"
       "86:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x13, #1, 87f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "tbz x13, #0, 88f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "tbz x14, #1, 87f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "tbz x14, #0, 88f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 88f\n"
       "87:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s17, [x23, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
       "88:"  // Height 3: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 90f\n"
       "89:"  // Height 3: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
       "90:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1103,36 +1103,36 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "92:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "93:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 94f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 95f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 95f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 95f\n"
       "94:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "95:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 98f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "blt 97f\n"
       "96:"  // Height 3: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
@@ -1140,65 +1140,65 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x12, #0x30]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "bge 96b\n"
       "97:"  // Height 3: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
@@ -1206,158 +1206,158 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       "98:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 103f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 103f\n"
+      "cmp x27, #0x4\n"
       "blt 100f\n"
       "99:"  // Height 3: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr q6, [x12, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q7, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "sub x26, x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "bge 99b\n"
       "100:"  // Height 3: Multiply loop: Skip odd blocks
-      "cbz x26, 103f\n"
-      "tbz x26, #1, 101f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "tbz x26, #0, 102f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
+      "cbz x27, 103f\n"
+      "tbz x27, #1, 101f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "tbz x27, #0, 102f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
       "b 102f\n"
       "101:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
       "102:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x11, x11, #0x20\n"
+      "ldr q6, [x11, #0x10]\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q7, [x10, #0x0]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q6, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "103:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 93b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
@@ -1367,10 +1367,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "tbz %x[flags], #1, 104f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v7.4s, v7.4s, v1.4s\n"
       "fmin v12.4s, v12.4s, v1.4s\n"
       "fmin v13.4s, v13.4s, v1.4s\n"
@@ -1396,140 +1396,140 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v18.4s, v18.4s, v0.4s\n"
       "fmax v19.4s, v19.4s, v0.4s\n"
       "104:"  // Height 3: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 113f\n"
-      "tbz x13, #3, 108f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "tbz x13, #2, 106f\n"
-      "st1 { v13.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 105f\n"
-      "str d14, [x12], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "tbz x13, #0, 112f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "tbz x14, #3, 108f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "tbz x14, #2, 106f\n"
+      "st1 { v13.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 105f\n"
+      "str d14, [x13], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "tbz x14, #0, 112f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 112f\n"
       "105:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 112f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "tbz x14, #0, 112f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 112f\n"
       "106:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 107f\n"
-      "str d13, [x12], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "tbz x13, #0, 112f\n"
-      "st1 { v13.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "tbz x14, #1, 107f\n"
+      "str d13, [x13], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "tbz x14, #0, 112f\n"
+      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 112f\n"
       "107:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 112f\n"
-      "str s13, [x12, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "tbz x14, #0, 112f\n"
+      "str s13, [x13, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 112f\n"
       "108:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 110f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 109f\n"
-      "str d12, [x12], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "tbz x13, #0, 112f\n"
-      "st1 { v12.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "tbz x14, #2, 110f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 109f\n"
+      "str d12, [x13], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "tbz x14, #0, 112f\n"
+      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 112f\n"
       "109:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 112f\n"
-      "str s12, [x12, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "tbz x14, #0, 112f\n"
+      "str s12, [x13, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 112f\n"
       "110:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 111f\n"
-      "str d7, [x12], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "tbz x13, #0, 112f\n"
-      "st1 { v7.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "tbz x14, #1, 111f\n"
+      "str d7, [x13], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "tbz x14, #0, 112f\n"
+      "st1 { v7.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 112f\n"
       "111:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s7, [x12, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s7, [x13, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "112:"  // Height 3: Partial direct writeback: Done
       "b 114f\n"
       "113:"  // Height 3: Full writeback
-      "str q7, [x12, #0x0]\n"
-      "str q12, [x12, #0x10]\n"
-      "str q13, [x12, #0x20]\n"
-      "str q14, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q7, [x13, #0x0]\n"
+      "str q12, [x13, #0x10]\n"
+      "str q13, [x13, #0x20]\n"
+      "str q14, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "114:"  // Height 3: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 78b\n"
       "b 230f\n"
       "115:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "116:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 117f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 117f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 117f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "117:"  // Height 4: B setup done
-      "cbz x14, 118f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 118f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "mov v16.16b, v8.16b\n"
@@ -1543,132 +1543,132 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "b 130f\n"
       "118:"  // Height 4: no bias
       "tbz %x[flags], #0, 129f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 127f\n"
-      "tbz x13, #3, 122f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 120f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 119f\n"
-      "ldr d16, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "tbz x13, #0, 126f\n"
-      "ld1 { v16.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "tbz x14, #3, 122f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 120f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 119f\n"
+      "ldr d16, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "tbz x14, #0, 126f\n"
+      "ld1 { v16.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 126f\n"
       "119:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 126f\n"
-      "ldr s16, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 126f\n"
+      "ldr s16, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 126f\n"
       "120:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x13, #1, 121f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "tbz x13, #0, 126f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "tbz x14, #1, 121f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "tbz x14, #0, 126f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 126f\n"
       "121:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 126f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 126f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 126f\n"
       "122:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x13, #2, 124f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 123f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "tbz x13, #0, 126f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "tbz x14, #2, 124f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 123f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "tbz x14, #0, 126f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 126f\n"
       "123:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 126f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 126f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 126f\n"
       "124:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x13, #1, 125f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "tbz x13, #0, 126f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "tbz x14, #1, 125f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "tbz x14, #0, 126f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 126f\n"
       "125:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "126:"  // Height 4: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 128f\n"
       "127:"  // Height 4: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "128:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1705,272 +1705,272 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "130:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "131:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 132f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 133f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 133f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 133f\n"
       "132:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "133:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 136f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "blt 135f\n"
       "134:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
-      "add x22, x22, #0x10\n"
-      "ldr q4, [x22, #0x0]\n"
+      "add x23, x23, #0x10\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "bge 134b\n"
       "135:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       "136:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 141f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 141f\n"
+      "cmp x27, #0x4\n"
       "blt 138f\n"
       "137:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d3, [x23], #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "cmp x26, #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x11, #0x10]\n"
+      "cmp x27, #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x11, x11, #0x20\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q7, [x10, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x9, #0x0]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q7, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "bge 137b\n"
       "138:"  // Height 4: Multiply loop: Skip odd blocks
-      "cbz x26, 141f\n"
-      "tbz x26, #1, 139f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "tbz x26, #0, 140f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
+      "cbz x27, 141f\n"
+      "tbz x27, #1, 139f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "tbz x27, #0, 140f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
       "b 140f\n"
       "139:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
       "140:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x0]\n"
+      "ldr q6, [x12, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x11, x11, #0x20\n"
+      "ldr q7, [x11, #0x0]\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q6, [x11, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q6, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "141:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 131b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
@@ -1985,10 +1985,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "tbz %x[flags], #1, 142f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v7.4s, v7.4s, v1.4s\n"
       "fmin v12.4s, v12.4s, v1.4s\n"
       "fmin v13.4s, v13.4s, v1.4s\n"
@@ -2022,160 +2022,160 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v18.4s, v18.4s, v0.4s\n"
       "fmax v19.4s, v19.4s, v0.4s\n"
       "142:"  // Height 4: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 151f\n"
-      "tbz x13, #3, 146f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 144f\n"
-      "st1 { v13.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 143f\n"
-      "str d14, [x12], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x13, #0, 150f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x14, #3, 146f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 144f\n"
+      "st1 { v13.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 143f\n"
+      "str d14, [x13], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x14, #0, 150f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 150f\n"
       "143:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 150f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x14, #0, 150f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 150f\n"
       "144:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 145f\n"
-      "str d13, [x12], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x13, #0, 150f\n"
-      "st1 { v13.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x14, #1, 145f\n"
+      "str d13, [x13], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x14, #0, 150f\n"
+      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 150f\n"
       "145:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 150f\n"
-      "str s13, [x12, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x14, #0, 150f\n"
+      "str s13, [x13, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 150f\n"
       "146:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 148f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 147f\n"
-      "str d12, [x12], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x13, #0, 150f\n"
-      "st1 { v12.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x14, #2, 148f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 147f\n"
+      "str d12, [x13], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x14, #0, 150f\n"
+      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 150f\n"
       "147:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 150f\n"
-      "str s12, [x12, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x14, #0, 150f\n"
+      "str s12, [x13, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 150f\n"
       "148:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 149f\n"
-      "str d7, [x12], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x13, #0, 150f\n"
-      "st1 { v7.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x14, #1, 149f\n"
+      "str d7, [x13], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x14, #0, 150f\n"
+      "st1 { v7.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 150f\n"
       "149:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s7, [x12, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s7, [x13, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "150:"  // Height 4: Partial direct writeback: Done
       "b 152f\n"
       "151:"  // Height 4: Full writeback
-      "str q7, [x12, #0x0]\n"
-      "str q12, [x12, #0x10]\n"
-      "str q13, [x12, #0x20]\n"
-      "str q14, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q7, [x13, #0x0]\n"
+      "str q12, [x13, #0x10]\n"
+      "str q13, [x13, #0x20]\n"
+      "str q14, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "152:"  // Height 4: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 116b\n"
       "b 230f\n"
       "153:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "154:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 155f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 155f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 155f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "155:"  // Height 5: B setup done
-      "cbz x14, 156f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 156f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "mov v16.16b, v8.16b\n"
@@ -2197,153 +2197,153 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "b 168f\n"
       "156:"  // Height 5: no bias
       "tbz %x[flags], #0, 167f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 165f\n"
-      "tbz x13, #3, 160f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 158f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 157f\n"
-      "ldr d16, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
-      "tbz x13, #0, 164f\n"
-      "ld1 { v16.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
+      "tbz x14, #3, 160f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 158f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 157f\n"
+      "ldr d16, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
+      "tbz x14, #0, 164f\n"
+      "ld1 { v16.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
       "b 164f\n"
       "157:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 164f\n"
-      "ldr s16, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 164f\n"
+      "ldr s16, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
       "b 164f\n"
       "158:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x13, #1, 159f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "tbz x13, #0, 164f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "tbz x14, #1, 159f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "tbz x14, #0, 164f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 164f\n"
       "159:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 164f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 164f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 164f\n"
       "160:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x13, #2, 162f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 161f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "tbz x13, #0, 164f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "tbz x14, #2, 162f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 161f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "tbz x14, #0, 164f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 164f\n"
       "161:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 164f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 164f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 164f\n"
       "162:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x13, #1, 163f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "tbz x13, #0, 164f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "tbz x14, #1, 163f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "tbz x14, #0, 164f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 164f\n"
       "163:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "164:"  // Height 5: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 166f\n"
       "165:"  // Height 5: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
       "166:"  // Height 5: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2396,43 +2396,43 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "168:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "169:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 170f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 171f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 171f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 171f\n"
       "170:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "171:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 174f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x21, #0x0]\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       "blt 173f\n"
       "172:"  // Height 5: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
@@ -2441,86 +2441,86 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x10, #0x0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x12, #0x30]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q7, [x11, #0x20]\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbc  // bfmmla v28.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb9  // bfmmla v25.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbd  // bfmmla v29.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecba  // bfmmla v26.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbe  // bfmmla v30.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
-      "ldr q3, [x23, #0x0]\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
-      "ldr q5, [x21, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
       "bge 172b\n"
       "173:"  // Height 5: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
@@ -2529,73 +2529,73 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q7, [x10, #0x0]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbc  // bfmmla v28.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb9  // bfmmla v25.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbd  // bfmmla v29.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecba  // bfmmla v26.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbe  // bfmmla v30.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
@@ -2603,51 +2603,51 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
       "174:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 179f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 179f\n"
+      "cmp x27, #0x4\n"
       "blt 176f\n"
       "175:"  // Height 5: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr q6, [x11, #0x0]\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr q6, [x12, #0x0]\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q7, [x11, #0x10]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "cmp x26, #0x4\n"
-      "add x11, x11, #0x20\n"
+      "cmp x27, #0x4\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9d  // bfmmla v29.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9a  // bfmmla v26.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9e  // bfmmla v30.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
@@ -2655,60 +2655,60 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "bge 175b\n"
       "176:"  // Height 5: Multiply loop: Skip odd blocks
-      "cbz x26, 179f\n"
-      "tbz x26, #1, 177f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "tbz x26, #0, 178f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
-      "ld1 { v5.h }[2], [x21]\n"
+      "cbz x27, 179f\n"
+      "tbz x27, #1, 177f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "tbz x27, #0, 178f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
+      "ld1 { v5.h }[2], [x22]\n"
       "b 178f\n"
       "177:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
-      "ldr h5, [x21, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
+      "ldr h5, [x22, #0x0]\n"
       "178:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q6, [x11, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q6, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
@@ -2716,18 +2716,18 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "179:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 169b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
@@ -2746,10 +2746,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "uzp1 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "tbz %x[flags], #1, 180f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v7.4s, v7.4s, v1.4s\n"
       "fmin v12.4s, v12.4s, v1.4s\n"
       "fmin v13.4s, v13.4s, v1.4s\n"
@@ -2791,183 +2791,183 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v26.4s, v26.4s, v0.4s\n"
       "fmax v27.4s, v27.4s, v0.4s\n"
       "180:"  // Height 5: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 189f\n"
-      "tbz x13, #3, 184f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 182f\n"
-      "st1 { v13.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 181f\n"
-      "str d14, [x12], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "tbz x13, #0, 188f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "tbz x14, #3, 184f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 182f\n"
+      "st1 { v13.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 181f\n"
+      "str d14, [x13], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "tbz x14, #0, 188f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 188f\n"
       "181:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 188f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "tbz x14, #0, 188f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 188f\n"
       "182:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 183f\n"
-      "str d13, [x12], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "tbz x13, #0, 188f\n"
-      "st1 { v13.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "tbz x14, #1, 183f\n"
+      "str d13, [x13], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "tbz x14, #0, 188f\n"
+      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 188f\n"
       "183:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 188f\n"
-      "str s13, [x12, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "tbz x14, #0, 188f\n"
+      "str s13, [x13, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 188f\n"
       "184:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 186f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 185f\n"
-      "str d12, [x12], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "tbz x13, #0, 188f\n"
-      "st1 { v12.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "tbz x14, #2, 186f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 185f\n"
+      "str d12, [x13], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "tbz x14, #0, 188f\n"
+      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 188f\n"
       "185:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 188f\n"
-      "str s12, [x12, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "tbz x14, #0, 188f\n"
+      "str s12, [x13, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 188f\n"
       "186:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 187f\n"
-      "str d7, [x12], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "tbz x13, #0, 188f\n"
-      "st1 { v7.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "tbz x14, #1, 187f\n"
+      "str d7, [x13], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "tbz x14, #0, 188f\n"
+      "st1 { v7.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 188f\n"
       "187:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s7, [x12, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s7, [x13, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "188:"  // Height 5: Partial direct writeback: Done
       "b 190f\n"
       "189:"  // Height 5: Full writeback
-      "str q7, [x12, #0x0]\n"
-      "str q12, [x12, #0x10]\n"
-      "str q13, [x12, #0x20]\n"
-      "str q14, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q7, [x13, #0x0]\n"
+      "str q12, [x13, #0x10]\n"
+      "str q13, [x13, #0x20]\n"
+      "str q14, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "190:"  // Height 5: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 154b\n"
       "b 230f\n"
       "191:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x18\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "192:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 193f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 193f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 193f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "193:"  // Height 6: B setup done
-      "cbz x14, 194f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 194f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "mov v16.16b, v8.16b\n"
@@ -2989,174 +2989,174 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "b 206f\n"
       "194:"  // Height 6: no bias
       "tbz %x[flags], #0, 205f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 203f\n"
-      "tbz x13, #3, 198f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 196f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 195f\n"
-      "ldr d16, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "ld1 { v16.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "tbz x14, #3, 198f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 196f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 195f\n"
+      "ldr d16, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "ld1 { v16.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 202f\n"
-      "ldr s16, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 202f\n"
+      "ldr s16, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x13, #1, 197f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "tbz x14, #1, 197f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 202f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 202f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x13, #2, 200f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 199f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "tbz x14, #2, 200f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 199f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 202f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 202f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x13, #1, 201f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "tbz x14, #1, 201f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "202:"  // Height 6: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 204f\n"
       "203:"  // Height 6: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q16, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q16, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "204:"  // Height 6: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -3209,213 +3209,213 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "206:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "207:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 208f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 209f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 209f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 209f\n"
       "208:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "209:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 212f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q3, [x23, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x21, #0x0]\n"
-      "ldr q6, [x20, #0x0]\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       "blt 211f\n"
       "210:"  // Height 6: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q7, [x10, #0x0]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x12, #0x30]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q7, [x11, #0x20]\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbc  // bfmmla v28.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb9  // bfmmla v25.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbd  // bfmmla v29.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecba  // bfmmla v26.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbe  // bfmmla v30.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
-      "ldr q3, [x23, #0x0]\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
-      "ldr q5, [x21, #0x0]\n"
-      "ldr q6, [x20, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 210b\n"
       "211:"  // Height 6: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q7, [x10, #0x0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x22, x22, #0x10\n"
       "add x21, x21, #0x10\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x11, #0x20]\n"
+      "ldr q7, [x12, #0x20]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbc  // bfmmla v28.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q6, [x11, #0x30]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
-      "add x10, x10, #0x40\n"
+      "add x11, x11, #0x40\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb9  // bfmmla v25.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbd  // bfmmla v29.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecba  // bfmmla v26.4s, v5.8h, v7.8h\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbe  // bfmmla v30.4s, v5.8h, v6.8h\n"
-      "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
@@ -3423,51 +3423,51 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
       "212:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 217f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 217f\n"
+      "cmp x27, #0x4\n"
       "blt 214f\n"
       "213:"  // Height 6: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d3, [x23], #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "cmp x26, #0x4\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "cmp x27, #0x4\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x12, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q7, [x11, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9d  // bfmmla v29.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q7, [x10, #0x10]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9a  // bfmmla v26.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9e  // bfmmla v30.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q7, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
@@ -3476,85 +3476,85 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "bge 213b\n"
       "214:"  // Height 6: Multiply loop: Skip odd blocks
-      "cbz x26, 217f\n"
-      "tbz x26, #1, 215f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "ldr s6, [x20], #0x4\n"
-      "tbz x26, #0, 216f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
-      "ld1 { v5.h }[2], [x21]\n"
-      "ld1 { v6.h }[2], [x20]\n"
+      "cbz x27, 217f\n"
+      "tbz x27, #1, 215f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "ldr s6, [x21], #0x4\n"
+      "tbz x27, #0, 216f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
+      "ld1 { v5.h }[2], [x22]\n"
+      "ld1 { v6.h }[2], [x21]\n"
       "b 216f\n"
       "215:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
-      "ldr h5, [x21, #0x0]\n"
-      "ldr h6, [x20, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
+      "ldr h5, [x22, #0x0]\n"
+      "ldr h6, [x21, #0x0]\n"
       "216:"  // Height 6: Multiply loop: Ragged operand read: Done
-      "ldr q7, [x11, #0x0]\n"
+      "ldr q7, [x12, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x10]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "217:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 207b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
@@ -3575,10 +3575,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "uzp1 v30.2d, v27.2d, v31.2d\n"
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "tbz %x[flags], #1, 218f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v7.4s, v7.4s, v1.4s\n"
       "fmin v12.4s, v12.4s, v1.4s\n"
       "fmin v13.4s, v13.4s, v1.4s\n"
@@ -3628,178 +3628,178 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 (
       "fmax v26.4s, v26.4s, v0.4s\n"
       "fmax v27.4s, v27.4s, v0.4s\n"
       "218:"  // Height 6: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 227f\n"
-      "tbz x13, #3, 222f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 220f\n"
-      "st1 { v13.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v29.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 219f\n"
-      "str d14, [x12], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x13, #0, 226f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x14, #3, 222f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 220f\n"
+      "st1 { v13.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v29.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 219f\n"
+      "str d14, [x13], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x14, #0, 226f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 226f\n"
       "219:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 226f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x14, #0, 226f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 226f\n"
       "220:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 221f\n"
-      "str d13, [x12], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d29, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x13, #0, 226f\n"
-      "st1 { v13.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v29.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x14, #1, 221f\n"
+      "str d13, [x13], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d29, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x14, #0, 226f\n"
+      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 226f\n"
       "221:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 226f\n"
-      "str s13, [x12, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s29, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x14, #0, 226f\n"
+      "str s13, [x13, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s29, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 226f\n"
       "222:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 224f\n"
-      "st1 { v7.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 223f\n"
-      "str d12, [x12], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d28, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x13, #0, 226f\n"
-      "st1 { v12.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v28.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x14, #2, 224f\n"
+      "st1 { v7.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 223f\n"
+      "str d12, [x13], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d28, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x14, #0, 226f\n"
+      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v28.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 226f\n"
       "223:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 226f\n"
-      "str s12, [x12, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s28, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x14, #0, 226f\n"
+      "str s12, [x13, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s28, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 226f\n"
       "224:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 225f\n"
-      "str d7, [x12], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x13, #0, 226f\n"
-      "st1 { v7.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x14, #1, 225f\n"
+      "str d7, [x13], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x14, #0, 226f\n"
+      "st1 { v7.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 226f\n"
       "225:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s7, [x12, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s7, [x13, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "226:"  // Height 6: Partial direct writeback: Done
       "b 228f\n"
       "227:"  // Height 6: Full writeback
-      "str q7, [x12, #0x0]\n"
-      "str q12, [x12, #0x10]\n"
-      "str q13, [x12, #0x20]\n"
-      "str q14, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q23, [x21, #0x0]\n"
-      "str q28, [x21, #0x10]\n"
-      "str q29, [x21, #0x20]\n"
-      "str q30, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q7, [x13, #0x0]\n"
+      "str q12, [x13, #0x10]\n"
+      "str q13, [x13, #0x20]\n"
+      "str q14, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q23, [x22, #0x0]\n"
+      "str q28, [x22, #0x10]\n"
+      "str q29, [x22, #0x20]\n"
+      "str q30, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "228:"  // Height 6: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 192b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 230f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 229f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "229:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "230:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp
index e1458b39ab..18a2db5069 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
 
@@ -104,156 +104,156 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "cmp %x[M], #0x2\n"
       "bgt 101f\n"
       "beq 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 3f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "cbz x14, 4f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
-      "add x14, x14, #0x40\n"
+      "cbz x15, 4f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
+      "add x15, x15, #0x40\n"
       "b 23f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 22f\n"
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 21f\n"
-      "tbz x13, #4, 12f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "tbz x13, #3, 8f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "tbz x13, #2, 6f\n"
-      "ldr d11, [x12], #0x8\n"
-      "tbz x13, #1, 5f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "mov x19, #0x3c\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v11.h }[6], [x12]\n"
+      "tbz x14, #4, 12f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "tbz x14, #3, 8f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "tbz x14, #2, 6f\n"
+      "ldr d11, [x13], #0x8\n"
+      "tbz x14, #1, 5f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "mov x20, #0x3c\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v11.h }[6], [x13]\n"
       "b 20f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v11.h }[4], [x12]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v11.h }[4], [x13]\n"
       "b 20f\n"
       "6:"  // Height 1: Partial accumulate: partial_2_24
-      "tbz x13, #1, 7f\n"
-      "ldr s11, [x12], #0x4\n"
-      "mov x19, #0x34\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v11.h }[2], [x12]\n"
+      "tbz x14, #1, 7f\n"
+      "ldr s11, [x13], #0x4\n"
+      "mov x20, #0x34\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v11.h }[2], [x13]\n"
       "b 20f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 20f\n"
-      "ldr h11, [x12, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 20f\n"
+      "ldr h11, [x13, #0x0]\n"
       "b 20f\n"
       "8:"  // Height 1: Partial accumulate: partial_4_16
-      "tbz x13, #2, 10f\n"
-      "ldr d10, [x12], #0x8\n"
-      "tbz x13, #1, 9f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "mov x19, #0x2c\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v10.h }[6], [x12]\n"
+      "tbz x14, #2, 10f\n"
+      "ldr d10, [x13], #0x8\n"
+      "tbz x14, #1, 9f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "mov x20, #0x2c\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v10.h }[6], [x13]\n"
       "b 20f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v10.h }[4], [x12]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v10.h }[4], [x13]\n"
       "b 20f\n"
       "10:"  // Height 1: Partial accumulate: partial_2_16
-      "tbz x13, #1, 11f\n"
-      "ldr s10, [x12], #0x4\n"
-      "mov x19, #0x24\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v10.h }[2], [x12]\n"
+      "tbz x14, #1, 11f\n"
+      "ldr s10, [x13], #0x4\n"
+      "mov x20, #0x24\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v10.h }[2], [x13]\n"
       "b 20f\n"
       "11:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 20f\n"
-      "ldr h10, [x12, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 20f\n"
+      "ldr h10, [x13, #0x0]\n"
       "b 20f\n"
       "12:"  // Height 1: Partial accumulate: partial_8_0
-      "tbz x13, #3, 16f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "tbz x13, #2, 14f\n"
-      "ldr d9, [x12], #0x8\n"
-      "tbz x13, #1, 13f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "mov x19, #0x1c\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v9.h }[6], [x12]\n"
+      "tbz x14, #3, 16f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "tbz x14, #2, 14f\n"
+      "ldr d9, [x13], #0x8\n"
+      "tbz x14, #1, 13f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "mov x20, #0x1c\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v9.h }[6], [x13]\n"
       "b 20f\n"
       "13:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v9.h }[4], [x12]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v9.h }[4], [x13]\n"
       "b 20f\n"
       "14:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x13, #1, 15f\n"
-      "ldr s9, [x12], #0x4\n"
-      "mov x19, #0x14\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v9.h }[2], [x12]\n"
+      "tbz x14, #1, 15f\n"
+      "ldr s9, [x13], #0x4\n"
+      "mov x20, #0x14\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v9.h }[2], [x13]\n"
       "b 20f\n"
       "15:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 20f\n"
-      "ldr h9, [x12, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 20f\n"
+      "ldr h9, [x13, #0x0]\n"
       "b 20f\n"
       "16:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x13, #2, 18f\n"
-      "ldr d8, [x12], #0x8\n"
-      "tbz x13, #1, 17f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "mov x19, #0xc\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v8.h }[6], [x12]\n"
+      "tbz x14, #2, 18f\n"
+      "ldr d8, [x13], #0x8\n"
+      "tbz x14, #1, 17f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "mov x20, #0xc\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v8.h }[6], [x13]\n"
       "b 20f\n"
       "17:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v8.h }[4], [x12]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v8.h }[4], [x13]\n"
       "b 20f\n"
       "18:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x13, #1, 19f\n"
-      "ldr s8, [x12], #0x4\n"
-      "mov x19, #0x4\n"
-      "tbz x13, #0, 20f\n"
-      "ld1 { v8.h }[2], [x12]\n"
+      "tbz x14, #1, 19f\n"
+      "ldr s8, [x13], #0x4\n"
+      "mov x20, #0x4\n"
+      "tbz x14, #0, 20f\n"
+      "ld1 { v8.h }[2], [x13]\n"
       "b 20f\n"
       "19:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x13, #0x0]\n"
+      "mov x20, #0x0\n"
       "20:"  // Height 1: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 23f\n"
       "21:"  // Height 1: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
       "b 23f\n"
       "22:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -261,200 +261,200 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "23:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "24:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 25f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 26f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 26f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 26f\n"
       "25:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "26:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 29f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 28f\n"
       "27:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
+      "ldr q7, [x11, #0x70]\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "ldr q7, [x9, #0x70]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "add x25, x25, #0x10\n"
-      "ldr q0, [x25, #0x0]\n"
+      "add x26, x26, #0x10\n"
+      "ldr q0, [x26, #0x0]\n"
+      "add x12, x12, #0x80\n"
+      "ldr q6, [x12, #0x0]\n"
       "add x11, x11, #0x80\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "add x10, x10, #0x80\n"
-      "ldr q7, [x10, #0x0]\n"
       "add x9, x9, #0x80\n"
-      "add x28, x28, #0x80\n"
       "bge 27b\n"
       "28:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
+      "ldr q7, [x11, #0x70]\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "sub x26, x26, #0x8\n"
+      "ldr q7, [x9, #0x70]\n"
+      "sub x27, x27, #0x8\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x12, x12, #0x80\n"
       "add x11, x11, #0x80\n"
       "add x10, x10, #0x80\n"
       "add x9, x9, #0x80\n"
-      "add x28, x28, #0x80\n"
       "29:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 31f\n"
+      "cbz x27, 31f\n"
       "30:"  // Height 1: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "sub x26, x26, #0x1\n"
-      "ldr q7, [x10, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "sub x27, x27, #0x1\n"
+      "ldr q7, [x11, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
       "add x9, x9, #0x10\n"
-      "add x28, x28, #0x10\n"
-      "cbnz x26, 30b\n"
+      "cbnz x27, 30b\n"
       "31:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 24b\n"
       "tbz %x[flags], #1, 32f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -464,305 +464,305 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v10.8h, v10.8h, v0.8h\n"
       "fmax v11.8h, v11.8h, v0.8h\n"
       "32:"  // Height 1: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 49f\n"
-      "tbz x13, #4, 40f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "tbz x13, #3, 36f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "tbz x13, #2, 34f\n"
-      "str d11, [x12], #0x8\n"
-      "tbz x13, #1, 33f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v11.h }[6], [x12]\n"
+      "tbz x14, #4, 40f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "tbz x14, #3, 36f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "tbz x14, #2, 34f\n"
+      "str d11, [x13], #0x8\n"
+      "tbz x14, #1, 33f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v11.h }[6], [x13]\n"
       "b 48f\n"
       "33:"  // Height 1: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 48f\n"
-      "st1 { v11.h }[4], [x12]\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v11.h }[4], [x13]\n"
       "b 48f\n"
       "34:"  // Height 1: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 35f\n"
-      "str s11, [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v11.h }[2], [x12]\n"
+      "tbz x14, #1, 35f\n"
+      "str s11, [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v11.h }[2], [x13]\n"
       "b 48f\n"
       "35:"  // Height 1: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 48f\n"
-      "str h11, [x12, #0x0]\n"
+      "tbz x14, #0, 48f\n"
+      "str h11, [x13, #0x0]\n"
       "b 48f\n"
       "36:"  // Height 1: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 38f\n"
-      "str d10, [x12], #0x8\n"
-      "tbz x13, #1, 37f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v10.h }[6], [x12]\n"
+      "tbz x14, #2, 38f\n"
+      "str d10, [x13], #0x8\n"
+      "tbz x14, #1, 37f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v10.h }[6], [x13]\n"
       "b 48f\n"
       "37:"  // Height 1: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 48f\n"
-      "st1 { v10.h }[4], [x12]\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v10.h }[4], [x13]\n"
       "b 48f\n"
       "38:"  // Height 1: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 39f\n"
-      "str s10, [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v10.h }[2], [x12]\n"
+      "tbz x14, #1, 39f\n"
+      "str s10, [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v10.h }[2], [x13]\n"
       "b 48f\n"
       "39:"  // Height 1: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 48f\n"
-      "str h10, [x12, #0x0]\n"
+      "tbz x14, #0, 48f\n"
+      "str h10, [x13, #0x0]\n"
       "b 48f\n"
       "40:"  // Height 1: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 44f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "tbz x13, #2, 42f\n"
-      "str d9, [x12], #0x8\n"
-      "tbz x13, #1, 41f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v9.h }[6], [x12]\n"
+      "tbz x14, #3, 44f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "tbz x14, #2, 42f\n"
+      "str d9, [x13], #0x8\n"
+      "tbz x14, #1, 41f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v9.h }[6], [x13]\n"
       "b 48f\n"
       "41:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 48f\n"
-      "st1 { v9.h }[4], [x12]\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v9.h }[4], [x13]\n"
       "b 48f\n"
       "42:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 43f\n"
-      "str s9, [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v9.h }[2], [x12]\n"
+      "tbz x14, #1, 43f\n"
+      "str s9, [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v9.h }[2], [x13]\n"
       "b 48f\n"
       "43:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 48f\n"
-      "str h9, [x12, #0x0]\n"
+      "tbz x14, #0, 48f\n"
+      "str h9, [x13, #0x0]\n"
       "b 48f\n"
       "44:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 46f\n"
-      "str d8, [x12], #0x8\n"
-      "tbz x13, #1, 45f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v8.h }[6], [x12]\n"
+      "tbz x14, #2, 46f\n"
+      "str d8, [x13], #0x8\n"
+      "tbz x14, #1, 45f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v8.h }[6], [x13]\n"
       "b 48f\n"
       "45:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 48f\n"
-      "st1 { v8.h }[4], [x12]\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v8.h }[4], [x13]\n"
       "b 48f\n"
       "46:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 47f\n"
-      "str s8, [x12], #0x4\n"
-      "tbz x13, #0, 48f\n"
-      "st1 { v8.h }[2], [x12]\n"
+      "tbz x14, #1, 47f\n"
+      "str s8, [x13], #0x4\n"
+      "tbz x14, #0, 48f\n"
+      "st1 { v8.h }[2], [x13]\n"
       "b 48f\n"
       "47:"  // Height 1: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
       "48:"  // Height 1: Partial direct writeback: Done
       "b 50f\n"
       "49:"  // Height 1: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
       "50:"  // Height 1: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 2b\n"
       "b 302f\n"
       "51:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "52:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 53f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 53f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 53f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "53:"  // Height 2: B setup done
-      "cbz x14, 54f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 54f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "b 73f\n"
       "54:"  // Height 2: no bias
       "tbz %x[flags], #0, 72f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x13, #0x20\n"
-      "add x24, x12, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x14, #0x20\n"
+      "add x25, x13, x20, LSL #1\n"
       "bge 71f\n"
-      "tbz x13, #4, 62f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "tbz x13, #3, 58f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "tbz x13, #2, 56f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "tbz x13, #1, 55f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "mov x19, #0x3c\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v11.h }[6], [x12]\n"
-      "ld1 { v15.h }[6], [x24]\n"
+      "tbz x14, #4, 62f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "tbz x14, #3, 58f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "tbz x14, #2, 56f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "tbz x14, #1, 55f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v11.h }[6], [x13]\n"
+      "ld1 { v15.h }[6], [x25]\n"
       "b 70f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v11.h }[4], [x12]\n"
-      "ld1 { v15.h }[4], [x24]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v11.h }[4], [x13]\n"
+      "ld1 { v15.h }[4], [x25]\n"
       "b 70f\n"
       "56:"  // Height 2: Partial accumulate: partial_2_24
-      "tbz x13, #1, 57f\n"
-      "ldr s11, [x12], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v11.h }[2], [x12]\n"
-      "ld1 { v15.h }[2], [x24]\n"
+      "tbz x14, #1, 57f\n"
+      "ldr s11, [x13], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v11.h }[2], [x13]\n"
+      "ld1 { v15.h }[2], [x25]\n"
       "b 70f\n"
       "57:"  // Height 2: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 70f\n"
-      "ldr h11, [x12, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 70f\n"
+      "ldr h11, [x13, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
       "b 70f\n"
       "58:"  // Height 2: Partial accumulate: partial_4_16
-      "tbz x13, #2, 60f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "tbz x13, #1, 59f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "mov x19, #0x2c\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v10.h }[6], [x12]\n"
-      "ld1 { v14.h }[6], [x24]\n"
+      "tbz x14, #2, 60f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "tbz x14, #1, 59f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v10.h }[6], [x13]\n"
+      "ld1 { v14.h }[6], [x25]\n"
       "b 70f\n"
       "59:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v10.h }[4], [x12]\n"
-      "ld1 { v14.h }[4], [x24]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v10.h }[4], [x13]\n"
+      "ld1 { v14.h }[4], [x25]\n"
       "b 70f\n"
       "60:"  // Height 2: Partial accumulate: partial_2_16
-      "tbz x13, #1, 61f\n"
-      "ldr s10, [x12], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v10.h }[2], [x12]\n"
-      "ld1 { v14.h }[2], [x24]\n"
+      "tbz x14, #1, 61f\n"
+      "ldr s10, [x13], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v10.h }[2], [x13]\n"
+      "ld1 { v14.h }[2], [x25]\n"
       "b 70f\n"
       "61:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 70f\n"
-      "ldr h10, [x12, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 70f\n"
+      "ldr h10, [x13, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
       "b 70f\n"
       "62:"  // Height 2: Partial accumulate: partial_8_0
-      "tbz x13, #3, 66f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "tbz x13, #2, 64f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "tbz x13, #1, 63f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "mov x19, #0x1c\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v9.h }[6], [x12]\n"
-      "ld1 { v13.h }[6], [x24]\n"
+      "tbz x14, #3, 66f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "tbz x14, #2, 64f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "tbz x14, #1, 63f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v9.h }[6], [x13]\n"
+      "ld1 { v13.h }[6], [x25]\n"
       "b 70f\n"
       "63:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v9.h }[4], [x12]\n"
-      "ld1 { v13.h }[4], [x24]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v9.h }[4], [x13]\n"
+      "ld1 { v13.h }[4], [x25]\n"
       "b 70f\n"
       "64:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x13, #1, 65f\n"
-      "ldr s9, [x12], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v9.h }[2], [x12]\n"
-      "ld1 { v13.h }[2], [x24]\n"
+      "tbz x14, #1, 65f\n"
+      "ldr s9, [x13], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v9.h }[2], [x13]\n"
+      "ld1 { v13.h }[2], [x25]\n"
       "b 70f\n"
       "65:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 70f\n"
-      "ldr h9, [x12, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 70f\n"
+      "ldr h9, [x13, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
       "b 70f\n"
       "66:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x13, #2, 68f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "tbz x13, #1, 67f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "mov x19, #0xc\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v8.h }[6], [x12]\n"
-      "ld1 { v12.h }[6], [x24]\n"
+      "tbz x14, #2, 68f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "tbz x14, #1, 67f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v8.h }[6], [x13]\n"
+      "ld1 { v12.h }[6], [x25]\n"
       "b 70f\n"
       "67:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v8.h }[4], [x12]\n"
-      "ld1 { v12.h }[4], [x24]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v8.h }[4], [x13]\n"
+      "ld1 { v12.h }[4], [x25]\n"
       "b 70f\n"
       "68:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x13, #1, 69f\n"
-      "ldr s8, [x12], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "tbz x13, #0, 70f\n"
-      "ld1 { v8.h }[2], [x12]\n"
-      "ld1 { v12.h }[2], [x24]\n"
+      "tbz x14, #1, 69f\n"
+      "ldr s8, [x13], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "tbz x14, #0, 70f\n"
+      "ld1 { v8.h }[2], [x13]\n"
+      "ld1 { v12.h }[2], [x25]\n"
       "b 70f\n"
       "69:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "ldr h12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x13, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "70:"  // Height 2: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 73f\n"
       "71:"  // Height 2: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "b 73f\n"
       "72:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -774,278 +774,278 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "73:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "74:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 75f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 76f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 76f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 76f\n"
       "75:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "76:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 79f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 78f\n"
       "77:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "sub x26, x26, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x8\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "cmp x27, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x12, #0x10]\n"
+      "add x26, x26, #0x10\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x11, #0x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 77b\n"
       "78:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "sub x26, x26, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x8\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x12, #0x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "79:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 81f\n"
+      "cbz x27, 81f\n"
       "80:"  // Height 2: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
-      "add x10, x10, #0x10\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
+      "add x10, x10, #0x10\n"
       "add x9, x9, #0x10\n"
-      "add x28, x28, #0x10\n"
-      "cbnz x26, 80b\n"
+      "cbnz x27, 80b\n"
       "81:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 74b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
       "tbz %x[flags], #1, 82f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -1063,382 +1063,382 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v14.8h, v14.8h, v0.8h\n"
       "fmax v15.8h, v15.8h, v0.8h\n"
       "82:"  // Height 2: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 99f\n"
-      "tbz x13, #4, 90f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "tbz x13, #3, 86f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "tbz x13, #2, 84f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "tbz x13, #1, 83f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v11.h }[6], [x12]\n"
-      "st1 { v15.h }[6], [x24]\n"
+      "tbz x14, #4, 90f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "tbz x14, #3, 86f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "tbz x14, #2, 84f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "tbz x14, #1, 83f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v11.h }[6], [x13]\n"
+      "st1 { v15.h }[6], [x25]\n"
       "b 98f\n"
       "83:"  // Height 2: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 98f\n"
-      "st1 { v11.h }[4], [x12]\n"
-      "st1 { v15.h }[4], [x24]\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v11.h }[4], [x13]\n"
+      "st1 { v15.h }[4], [x25]\n"
       "b 98f\n"
       "84:"  // Height 2: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 85f\n"
-      "str s11, [x12], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v11.h }[2], [x12]\n"
-      "st1 { v15.h }[2], [x24]\n"
+      "tbz x14, #1, 85f\n"
+      "str s11, [x13], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v11.h }[2], [x13]\n"
+      "st1 { v15.h }[2], [x25]\n"
       "b 98f\n"
       "85:"  // Height 2: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 98f\n"
-      "str h11, [x12, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
+      "tbz x14, #0, 98f\n"
+      "str h11, [x13, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
       "b 98f\n"
       "86:"  // Height 2: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 88f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "tbz x13, #1, 87f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v10.h }[6], [x12]\n"
-      "st1 { v14.h }[6], [x24]\n"
+      "tbz x14, #2, 88f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "tbz x14, #1, 87f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v10.h }[6], [x13]\n"
+      "st1 { v14.h }[6], [x25]\n"
       "b 98f\n"
       "87:"  // Height 2: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 98f\n"
-      "st1 { v10.h }[4], [x12]\n"
-      "st1 { v14.h }[4], [x24]\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v10.h }[4], [x13]\n"
+      "st1 { v14.h }[4], [x25]\n"
       "b 98f\n"
       "88:"  // Height 2: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 89f\n"
-      "str s10, [x12], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v10.h }[2], [x12]\n"
-      "st1 { v14.h }[2], [x24]\n"
+      "tbz x14, #1, 89f\n"
+      "str s10, [x13], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v10.h }[2], [x13]\n"
+      "st1 { v14.h }[2], [x25]\n"
       "b 98f\n"
       "89:"  // Height 2: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 98f\n"
-      "str h10, [x12, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
+      "tbz x14, #0, 98f\n"
+      "str h10, [x13, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
       "b 98f\n"
       "90:"  // Height 2: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 94f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "tbz x13, #2, 92f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "tbz x13, #1, 91f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v9.h }[6], [x12]\n"
-      "st1 { v13.h }[6], [x24]\n"
+      "tbz x14, #3, 94f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "tbz x14, #2, 92f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "tbz x14, #1, 91f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v9.h }[6], [x13]\n"
+      "st1 { v13.h }[6], [x25]\n"
       "b 98f\n"
       "91:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 98f\n"
-      "st1 { v9.h }[4], [x12]\n"
-      "st1 { v13.h }[4], [x24]\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v9.h }[4], [x13]\n"
+      "st1 { v13.h }[4], [x25]\n"
       "b 98f\n"
       "92:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 93f\n"
-      "str s9, [x12], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v9.h }[2], [x12]\n"
-      "st1 { v13.h }[2], [x24]\n"
+      "tbz x14, #1, 93f\n"
+      "str s9, [x13], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v9.h }[2], [x13]\n"
+      "st1 { v13.h }[2], [x25]\n"
       "b 98f\n"
       "93:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 98f\n"
-      "str h9, [x12, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
+      "tbz x14, #0, 98f\n"
+      "str h9, [x13, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
       "b 98f\n"
       "94:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 96f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "tbz x13, #1, 95f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v8.h }[6], [x12]\n"
-      "st1 { v12.h }[6], [x24]\n"
+      "tbz x14, #2, 96f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "tbz x14, #1, 95f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v8.h }[6], [x13]\n"
+      "st1 { v12.h }[6], [x25]\n"
       "b 98f\n"
       "95:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 98f\n"
-      "st1 { v8.h }[4], [x12]\n"
-      "st1 { v12.h }[4], [x24]\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v8.h }[4], [x13]\n"
+      "st1 { v12.h }[4], [x25]\n"
       "b 98f\n"
       "96:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 97f\n"
-      "str s8, [x12], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "tbz x13, #0, 98f\n"
-      "st1 { v8.h }[2], [x12]\n"
-      "st1 { v12.h }[2], [x24]\n"
+      "tbz x14, #1, 97f\n"
+      "str s8, [x13], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "tbz x14, #0, 98f\n"
+      "st1 { v8.h }[2], [x13]\n"
+      "st1 { v12.h }[2], [x25]\n"
       "b 98f\n"
       "97:"  // Height 2: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
       "98:"  // Height 2: Partial direct writeback: Done
       "b 100f\n"
       "99:"  // Height 2: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
       "100:"  // Height 2: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 52b\n"
       "b 302f\n"
       "101:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "102:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 103f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 103f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 103f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "103:"  // Height 3: B setup done
-      "cbz x14, 104f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 104f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 123f\n"
       "104:"  // Height 3: no bias
       "tbz %x[flags], #0, 122f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "cmp x13, #0x20\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "cmp x14, #0x20\n"
+      "add x24, x25, x20, LSL #1\n"
       "bge 121f\n"
-      "tbz x13, #4, 112f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "tbz x13, #3, 108f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "tbz x13, #2, 106f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "tbz x13, #1, 105f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v11.h }[6], [x12]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
+      "tbz x14, #4, 112f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "tbz x14, #3, 108f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "tbz x14, #2, 106f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "tbz x14, #1, 105f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v11.h }[6], [x13]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
       "b 120f\n"
       "105:"  // Height 3: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v11.h }[4], [x12]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v11.h }[4], [x13]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
       "b 120f\n"
       "106:"  // Height 3: Partial accumulate: partial_2_24
-      "tbz x13, #1, 107f\n"
-      "ldr s11, [x12], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v11.h }[2], [x12]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
+      "tbz x14, #1, 107f\n"
+      "ldr s11, [x13], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v11.h }[2], [x13]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
       "b 120f\n"
       "107:"  // Height 3: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 120f\n"
-      "ldr h11, [x12, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 120f\n"
+      "ldr h11, [x13, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
       "b 120f\n"
       "108:"  // Height 3: Partial accumulate: partial_4_16
-      "tbz x13, #2, 110f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "tbz x13, #1, 109f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v10.h }[6], [x12]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
+      "tbz x14, #2, 110f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "tbz x14, #1, 109f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v10.h }[6], [x13]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
       "b 120f\n"
       "109:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v10.h }[4], [x12]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v10.h }[4], [x13]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
       "b 120f\n"
       "110:"  // Height 3: Partial accumulate: partial_2_16
-      "tbz x13, #1, 111f\n"
-      "ldr s10, [x12], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v10.h }[2], [x12]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
+      "tbz x14, #1, 111f\n"
+      "ldr s10, [x13], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v10.h }[2], [x13]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
       "b 120f\n"
       "111:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 120f\n"
-      "ldr h10, [x12, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 120f\n"
+      "ldr h10, [x13, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
       "b 120f\n"
       "112:"  // Height 3: Partial accumulate: partial_8_0
-      "tbz x13, #3, 116f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "tbz x13, #2, 114f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "tbz x13, #1, 113f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v9.h }[6], [x12]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
+      "tbz x14, #3, 116f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "tbz x14, #2, 114f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "tbz x14, #1, 113f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v9.h }[6], [x13]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
       "b 120f\n"
       "113:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v9.h }[4], [x12]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v9.h }[4], [x13]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
       "b 120f\n"
       "114:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x13, #1, 115f\n"
-      "ldr s9, [x12], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v9.h }[2], [x12]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
+      "tbz x14, #1, 115f\n"
+      "ldr s9, [x13], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v9.h }[2], [x13]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
       "b 120f\n"
       "115:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 120f\n"
-      "ldr h9, [x12, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 120f\n"
+      "ldr h9, [x13, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
       "b 120f\n"
       "116:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x13, #2, 118f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "tbz x13, #1, 117f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v8.h }[6], [x12]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
+      "tbz x14, #2, 118f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "tbz x14, #1, 117f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v8.h }[6], [x13]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
       "b 120f\n"
       "117:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v8.h }[4], [x12]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v8.h }[4], [x13]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
       "b 120f\n"
       "118:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x13, #1, 119f\n"
-      "ldr s8, [x12], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "tbz x13, #0, 120f\n"
-      "ld1 { v8.h }[2], [x12]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
+      "tbz x14, #1, 119f\n"
+      "ldr s8, [x13], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "tbz x14, #0, 120f\n"
+      "ld1 { v8.h }[2], [x13]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
       "b 120f\n"
       "119:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "ldr h12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h16, [x23, #0x0]\n"
+      "ldr h8, [x13, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
       "120:"  // Height 3: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 123f\n"
       "121:"  // Height 3: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
       "b 123f\n"
       "122:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1454,308 +1454,308 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "123:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "124:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 125f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 126f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 126f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 126f\n"
       "125:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "126:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 129f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 128f\n"
       "127:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "add x11, x11, #0x80\n"
+      "add x12, x12, #0x80\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "add x9, x9, #0x80\n"
+      "add x10, x10, #0x80\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 127b\n"
       "128:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "add x11, x11, #0x80\n"
+      "add x12, x12, #0x80\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "add x9, x9, #0x80\n"
+      "add x10, x10, #0x80\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
@@ -1763,46 +1763,46 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "129:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 131f\n"
+      "cbz x27, 131f\n"
       "130:"  // Height 3: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x11, x11, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
-      "add x9, x9, #0x10\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "cbnz x26, 130b\n"
+      "cbnz x27, 130b\n"
       "131:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 124b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "tbz %x[flags], #1, 132f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -1828,220 +1828,220 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v18.8h, v18.8h, v0.8h\n"
       "fmax v19.8h, v19.8h, v0.8h\n"
       "132:"  // Height 3: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 149f\n"
-      "tbz x13, #4, 140f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "tbz x13, #3, 136f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "tbz x13, #2, 134f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "tbz x13, #1, 133f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v11.h }[6], [x12]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
+      "tbz x14, #4, 140f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "tbz x14, #3, 136f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "tbz x14, #2, 134f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "tbz x14, #1, 133f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v11.h }[6], [x13]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
       "b 148f\n"
       "133:"  // Height 3: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 148f\n"
-      "st1 { v11.h }[4], [x12]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v11.h }[4], [x13]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
       "b 148f\n"
       "134:"  // Height 3: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 135f\n"
-      "str s11, [x12], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v11.h }[2], [x12]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
+      "tbz x14, #1, 135f\n"
+      "str s11, [x13], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v11.h }[2], [x13]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
       "b 148f\n"
       "135:"  // Height 3: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 148f\n"
-      "str h11, [x12, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
+      "tbz x14, #0, 148f\n"
+      "str h11, [x13, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
       "b 148f\n"
       "136:"  // Height 3: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 138f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "tbz x13, #1, 137f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v10.h }[6], [x12]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
+      "tbz x14, #2, 138f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "tbz x14, #1, 137f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v10.h }[6], [x13]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
       "b 148f\n"
       "137:"  // Height 3: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 148f\n"
-      "st1 { v10.h }[4], [x12]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v10.h }[4], [x13]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
       "b 148f\n"
       "138:"  // Height 3: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 139f\n"
-      "str s10, [x12], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v10.h }[2], [x12]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
+      "tbz x14, #1, 139f\n"
+      "str s10, [x13], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v10.h }[2], [x13]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
       "b 148f\n"
       "139:"  // Height 3: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 148f\n"
-      "str h10, [x12, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
+      "tbz x14, #0, 148f\n"
+      "str h10, [x13, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
       "b 148f\n"
       "140:"  // Height 3: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 144f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "tbz x13, #2, 142f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "tbz x13, #1, 141f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v9.h }[6], [x12]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
+      "tbz x14, #3, 144f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "tbz x14, #2, 142f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "tbz x14, #1, 141f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v9.h }[6], [x13]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
       "b 148f\n"
       "141:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 148f\n"
-      "st1 { v9.h }[4], [x12]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v9.h }[4], [x13]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
       "b 148f\n"
       "142:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 143f\n"
-      "str s9, [x12], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v9.h }[2], [x12]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
+      "tbz x14, #1, 143f\n"
+      "str s9, [x13], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v9.h }[2], [x13]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
       "b 148f\n"
       "143:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 148f\n"
-      "str h9, [x12, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
+      "tbz x14, #0, 148f\n"
+      "str h9, [x13, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
       "b 148f\n"
       "144:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 146f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "tbz x13, #1, 145f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v8.h }[6], [x12]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
+      "tbz x14, #2, 146f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "tbz x14, #1, 145f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v8.h }[6], [x13]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
       "b 148f\n"
       "145:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 148f\n"
-      "st1 { v8.h }[4], [x12]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v8.h }[4], [x13]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
       "b 148f\n"
       "146:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 147f\n"
-      "str s8, [x12], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "tbz x13, #0, 148f\n"
-      "st1 { v8.h }[2], [x12]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
+      "tbz x14, #1, 147f\n"
+      "str s8, [x13], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "tbz x14, #0, 148f\n"
+      "st1 { v8.h }[2], [x13]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
       "b 148f\n"
       "147:"  // Height 3: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
       "148:"  // Height 3: Partial direct writeback: Done
       "b 150f\n"
       "149:"  // Height 3: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "150:"  // Height 3: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 102b\n"
       "b 302f\n"
       "151:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "152:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 153f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 153f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 153f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "153:"  // Height 4: B setup done
-      "cbz x14, 154f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 154f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -2051,236 +2051,236 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "b 173f\n"
       "154:"  // Height 4: no bias
       "tbz %x[flags], #0, 172f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "cmp x13, #0x20\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "cmp x14, #0x20\n"
+      "add x23, x24, x20, LSL #1\n"
       "bge 171f\n"
-      "tbz x13, #4, 162f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
-      "tbz x13, #3, 158f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
-      "tbz x13, #2, 156f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "tbz x13, #1, 155f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v11.h }[6], [x12]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
+      "tbz x14, #4, 162f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
+      "tbz x14, #3, 158f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
+      "tbz x14, #2, 156f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "tbz x14, #1, 155f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v11.h }[6], [x13]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
       "b 170f\n"
       "155:"  // Height 4: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v11.h }[4], [x12]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v11.h }[4], [x13]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
       "b 170f\n"
       "156:"  // Height 4: Partial accumulate: partial_2_24
-      "tbz x13, #1, 157f\n"
-      "ldr s11, [x12], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v11.h }[2], [x12]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
+      "tbz x14, #1, 157f\n"
+      "ldr s11, [x13], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v11.h }[2], [x13]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
       "b 170f\n"
       "157:"  // Height 4: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 170f\n"
-      "ldr h11, [x12, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 170f\n"
+      "ldr h11, [x13, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
       "b 170f\n"
       "158:"  // Height 4: Partial accumulate: partial_4_16
-      "tbz x13, #2, 160f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "tbz x13, #1, 159f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v10.h }[6], [x12]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
+      "tbz x14, #2, 160f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "tbz x14, #1, 159f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v10.h }[6], [x13]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
       "b 170f\n"
       "159:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v10.h }[4], [x12]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v10.h }[4], [x13]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
       "b 170f\n"
       "160:"  // Height 4: Partial accumulate: partial_2_16
-      "tbz x13, #1, 161f\n"
-      "ldr s10, [x12], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v10.h }[2], [x12]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
+      "tbz x14, #1, 161f\n"
+      "ldr s10, [x13], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v10.h }[2], [x13]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
       "b 170f\n"
       "161:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 170f\n"
-      "ldr h10, [x12, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 170f\n"
+      "ldr h10, [x13, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
       "b 170f\n"
       "162:"  // Height 4: Partial accumulate: partial_8_0
-      "tbz x13, #3, 166f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "tbz x13, #2, 164f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "tbz x13, #1, 163f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v9.h }[6], [x12]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
+      "tbz x14, #3, 166f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "tbz x14, #2, 164f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "tbz x14, #1, 163f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v9.h }[6], [x13]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
       "b 170f\n"
       "163:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v9.h }[4], [x12]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v9.h }[4], [x13]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
       "b 170f\n"
       "164:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x13, #1, 165f\n"
-      "ldr s9, [x12], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v9.h }[2], [x12]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
+      "tbz x14, #1, 165f\n"
+      "ldr s9, [x13], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v9.h }[2], [x13]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
       "b 170f\n"
       "165:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 170f\n"
-      "ldr h9, [x12, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 170f\n"
+      "ldr h9, [x13, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
       "b 170f\n"
       "166:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x13, #2, 168f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "tbz x13, #1, 167f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v8.h }[6], [x12]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
+      "tbz x14, #2, 168f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "tbz x14, #1, 167f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v8.h }[6], [x13]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
       "b 170f\n"
       "167:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v8.h }[4], [x12]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v8.h }[4], [x13]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
       "b 170f\n"
       "168:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x13, #1, 169f\n"
-      "ldr s8, [x12], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
-      "tbz x13, #0, 170f\n"
-      "ld1 { v8.h }[2], [x12]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
+      "tbz x14, #1, 169f\n"
+      "ldr s8, [x13], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
+      "tbz x14, #0, 170f\n"
+      "ld1 { v8.h }[2], [x13]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
       "b 170f\n"
       "169:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "ldr h12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
+      "ldr h8, [x13, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
       "170:"  // Height 4: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 173f\n"
       "171:"  // Height 4: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "b 173f\n"
       "172:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2300,377 +2300,377 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "173:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "174:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 175f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 176f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 176f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 176f\n"
       "175:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "176:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 179f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 178f\n"
       "177:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x26, x26, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 177b\n"
       "178:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
@@ -2680,29 +2680,29 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "179:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 181f\n"
+      "cbz x27, 181f\n"
       "180:"  // Height 4: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x11, x11, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
+      "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
-      "add x9, x9, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
@@ -2711,21 +2711,21 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "cbnz x26, 180b\n"
+      "cbnz x27, 180b\n"
       "181:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 174b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "tbz %x[flags], #1, 182f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -2759,256 +2759,256 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v22.8h, v22.8h, v0.8h\n"
       "fmax v23.8h, v23.8h, v0.8h\n"
       "182:"  // Height 4: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 199f\n"
-      "tbz x13, #4, 190f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
-      "tbz x13, #3, 186f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
-      "tbz x13, #2, 184f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "tbz x13, #1, 183f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v11.h }[6], [x12]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
+      "tbz x14, #4, 190f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
+      "tbz x14, #3, 186f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
+      "tbz x14, #2, 184f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "tbz x14, #1, 183f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v11.h }[6], [x13]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
       "b 198f\n"
       "183:"  // Height 4: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 198f\n"
-      "st1 { v11.h }[4], [x12]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v11.h }[4], [x13]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
       "b 198f\n"
       "184:"  // Height 4: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 185f\n"
-      "str s11, [x12], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v11.h }[2], [x12]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
+      "tbz x14, #1, 185f\n"
+      "str s11, [x13], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v11.h }[2], [x13]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
       "b 198f\n"
       "185:"  // Height 4: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 198f\n"
-      "str h11, [x12, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
+      "tbz x14, #0, 198f\n"
+      "str h11, [x13, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
       "b 198f\n"
       "186:"  // Height 4: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 188f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "tbz x13, #1, 187f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v10.h }[6], [x12]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
+      "tbz x14, #2, 188f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "tbz x14, #1, 187f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v10.h }[6], [x13]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
       "b 198f\n"
       "187:"  // Height 4: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 198f\n"
-      "st1 { v10.h }[4], [x12]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v10.h }[4], [x13]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
       "b 198f\n"
       "188:"  // Height 4: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 189f\n"
-      "str s10, [x12], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v10.h }[2], [x12]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
+      "tbz x14, #1, 189f\n"
+      "str s10, [x13], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v10.h }[2], [x13]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
       "b 198f\n"
       "189:"  // Height 4: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 198f\n"
-      "str h10, [x12, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
+      "tbz x14, #0, 198f\n"
+      "str h10, [x13, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
       "b 198f\n"
       "190:"  // Height 4: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 194f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "tbz x13, #2, 192f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "tbz x13, #1, 191f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v9.h }[6], [x12]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
+      "tbz x14, #3, 194f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "tbz x14, #2, 192f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "tbz x14, #1, 191f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v9.h }[6], [x13]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
       "b 198f\n"
       "191:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 198f\n"
-      "st1 { v9.h }[4], [x12]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v9.h }[4], [x13]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
       "b 198f\n"
       "192:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 193f\n"
-      "str s9, [x12], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v9.h }[2], [x12]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
+      "tbz x14, #1, 193f\n"
+      "str s9, [x13], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v9.h }[2], [x13]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
       "b 198f\n"
       "193:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 198f\n"
-      "str h9, [x12, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
+      "tbz x14, #0, 198f\n"
+      "str h9, [x13, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
       "b 198f\n"
       "194:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 196f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "tbz x13, #1, 195f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v8.h }[6], [x12]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
+      "tbz x14, #2, 196f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "tbz x14, #1, 195f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v8.h }[6], [x13]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
       "b 198f\n"
       "195:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 198f\n"
-      "st1 { v8.h }[4], [x12]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v8.h }[4], [x13]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
       "b 198f\n"
       "196:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 197f\n"
-      "str s8, [x12], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
-      "tbz x13, #0, 198f\n"
-      "st1 { v8.h }[2], [x12]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
+      "tbz x14, #1, 197f\n"
+      "str s8, [x13], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
+      "tbz x14, #0, 198f\n"
+      "st1 { v8.h }[2], [x13]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
       "b 198f\n"
       "197:"  // Height 4: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
       "198:"  // Height 4: Partial direct writeback: Done
       "b 200f\n"
       "199:"  // Height 4: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
       "200:"  // Height 4: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 152b\n"
       "b 302f\n"
       "201:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "202:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 203f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 203f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 203f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "203:"  // Height 5: B setup done
-      "cbz x14, 204f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 204f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -3022,273 +3022,273 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "b 223f\n"
       "204:"  // Height 5: no bias
       "tbz %x[flags], #0, 222f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "cmp x13, #0x20\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "cmp x14, #0x20\n"
+      "add x22, x23, x20, LSL #1\n"
       "bge 221f\n"
-      "tbz x13, #4, 212f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
-      "ld1 { v25.8h }, [x21], #0x10\n"
-      "tbz x13, #3, 208f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
-      "ld1 { v26.8h }, [x21], #0x10\n"
-      "tbz x13, #2, 206f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "tbz x13, #1, 205f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
-      "ld1 { v27.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v11.h }[6], [x12]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
-      "ld1 { v27.h }[6], [x21]\n"
+      "tbz x14, #4, 212f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
+      "ld1 { v25.8h }, [x22], #0x10\n"
+      "tbz x14, #3, 208f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
+      "ld1 { v26.8h }, [x22], #0x10\n"
+      "tbz x14, #2, 206f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "tbz x14, #1, 205f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
+      "ld1 { v27.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v11.h }[6], [x13]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
+      "ld1 { v27.h }[6], [x22]\n"
       "b 220f\n"
       "205:"  // Height 5: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v11.h }[4], [x12]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
-      "ld1 { v27.h }[4], [x21]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v11.h }[4], [x13]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
+      "ld1 { v27.h }[4], [x22]\n"
       "b 220f\n"
       "206:"  // Height 5: Partial accumulate: partial_2_24
-      "tbz x13, #1, 207f\n"
-      "ldr s11, [x12], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
-      "ldr s27, [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v11.h }[2], [x12]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
-      "ld1 { v27.h }[2], [x21]\n"
+      "tbz x14, #1, 207f\n"
+      "ldr s11, [x13], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
+      "ldr s27, [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v11.h }[2], [x13]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
+      "ld1 { v27.h }[2], [x22]\n"
       "b 220f\n"
       "207:"  // Height 5: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 220f\n"
-      "ldr h11, [x12, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
-      "ldr h27, [x21, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 220f\n"
+      "ldr h11, [x13, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
+      "ldr h27, [x22, #0x0]\n"
       "b 220f\n"
       "208:"  // Height 5: Partial accumulate: partial_4_16
-      "tbz x13, #2, 210f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "tbz x13, #1, 209f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
-      "ld1 { v26.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v10.h }[6], [x12]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
-      "ld1 { v26.h }[6], [x21]\n"
+      "tbz x14, #2, 210f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "tbz x14, #1, 209f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
+      "ld1 { v26.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v10.h }[6], [x13]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
+      "ld1 { v26.h }[6], [x22]\n"
       "b 220f\n"
       "209:"  // Height 5: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v10.h }[4], [x12]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
-      "ld1 { v26.h }[4], [x21]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v10.h }[4], [x13]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
+      "ld1 { v26.h }[4], [x22]\n"
       "b 220f\n"
       "210:"  // Height 5: Partial accumulate: partial_2_16
-      "tbz x13, #1, 211f\n"
-      "ldr s10, [x12], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
-      "ldr s26, [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v10.h }[2], [x12]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
-      "ld1 { v26.h }[2], [x21]\n"
+      "tbz x14, #1, 211f\n"
+      "ldr s10, [x13], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
+      "ldr s26, [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v10.h }[2], [x13]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
+      "ld1 { v26.h }[2], [x22]\n"
       "b 220f\n"
       "211:"  // Height 5: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 220f\n"
-      "ldr h10, [x12, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
-      "ldr h26, [x21, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 220f\n"
+      "ldr h10, [x13, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
+      "ldr h26, [x22, #0x0]\n"
       "b 220f\n"
       "212:"  // Height 5: Partial accumulate: partial_8_0
-      "tbz x13, #3, 216f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "tbz x13, #2, 214f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "tbz x13, #1, 213f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
-      "ld1 { v25.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v9.h }[6], [x12]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
-      "ld1 { v25.h }[6], [x21]\n"
+      "tbz x14, #3, 216f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "tbz x14, #2, 214f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "tbz x14, #1, 213f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
+      "ld1 { v25.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v9.h }[6], [x13]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
+      "ld1 { v25.h }[6], [x22]\n"
       "b 220f\n"
       "213:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v9.h }[4], [x12]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
-      "ld1 { v25.h }[4], [x21]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v9.h }[4], [x13]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
+      "ld1 { v25.h }[4], [x22]\n"
       "b 220f\n"
       "214:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x13, #1, 215f\n"
-      "ldr s9, [x12], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
-      "ldr s25, [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v9.h }[2], [x12]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
-      "ld1 { v25.h }[2], [x21]\n"
+      "tbz x14, #1, 215f\n"
+      "ldr s9, [x13], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
+      "ldr s25, [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v9.h }[2], [x13]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
+      "ld1 { v25.h }[2], [x22]\n"
       "b 220f\n"
       "215:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 220f\n"
-      "ldr h9, [x12, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
-      "ldr h25, [x21, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 220f\n"
+      "ldr h9, [x13, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
+      "ldr h25, [x22, #0x0]\n"
       "b 220f\n"
       "216:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x13, #2, 218f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "tbz x13, #1, 217f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
-      "ld1 { v24.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v8.h }[6], [x12]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
-      "ld1 { v24.h }[6], [x21]\n"
+      "tbz x14, #2, 218f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "tbz x14, #1, 217f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
+      "ld1 { v24.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v8.h }[6], [x13]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
+      "ld1 { v24.h }[6], [x22]\n"
       "b 220f\n"
       "217:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v8.h }[4], [x12]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
-      "ld1 { v24.h }[4], [x21]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v8.h }[4], [x13]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
+      "ld1 { v24.h }[4], [x22]\n"
       "b 220f\n"
       "218:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x13, #1, 219f\n"
-      "ldr s8, [x12], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
-      "ldr s24, [x21], #0x4\n"
-      "tbz x13, #0, 220f\n"
-      "ld1 { v8.h }[2], [x12]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
-      "ld1 { v24.h }[2], [x21]\n"
+      "tbz x14, #1, 219f\n"
+      "ldr s8, [x13], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
+      "ldr s24, [x22], #0x4\n"
+      "tbz x14, #0, 220f\n"
+      "ld1 { v8.h }[2], [x13]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
+      "ld1 { v24.h }[2], [x22]\n"
       "b 220f\n"
       "219:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "ldr h12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
-      "ldr h24, [x21, #0x0]\n"
+      "ldr h8, [x13, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
+      "ldr h24, [x22, #0x0]\n"
       "220:"  // Height 5: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 223f\n"
       "221:"  // Height 5: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
       "b 223f\n"
       "222:"  // Height 5: no accumulate
       "movi v8.16b, #0x0\n"
@@ -3312,446 +3312,446 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "223:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "224:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 225f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 226f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 226f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 226f\n"
       "225:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "226:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 229f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 228f\n"
       "227:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "add x11, x11, #0x80\n"
+      "add x12, x12, #0x80\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "add x9, x9, #0x80\n"
+      "add x10, x10, #0x80\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "fmla v26.8h, v6.8h, v4.h[7]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 227b\n"
       "228:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
+      "ldr q6, [x12, #0x70]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "add x11, x11, #0x80\n"
+      "add x12, x12, #0x80\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "add x9, x9, #0x80\n"
+      "add x10, x10, #0x80\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
@@ -3763,32 +3763,32 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "229:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 231f\n"
+      "cbz x27, 231f\n"
       "230:"  // Height 5: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "add x11, x11, #0x10\n"
+      "add x12, x12, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x10, x10, #0x10\n"
+      "add x11, x11, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "add x9, x9, #0x10\n"
+      "add x10, x10, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
@@ -3799,22 +3799,22 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "cbnz x26, 230b\n"
+      "cbnz x27, 230b\n"
       "231:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 224b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "tbz %x[flags], #1, 232f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -3856,295 +3856,295 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v26.8h, v26.8h, v0.8h\n"
       "fmax v27.8h, v27.8h, v0.8h\n"
       "232:"  // Height 5: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 249f\n"
-      "tbz x13, #4, 240f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v25.8h }, [x21], #0x10\n"
-      "tbz x13, #3, 236f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
-      "st1 { v26.8h }, [x21], #0x10\n"
-      "tbz x13, #2, 234f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "tbz x13, #1, 233f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
-      "st1 { v27.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v11.h }[6], [x12]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
-      "st1 { v27.h }[6], [x21]\n"
+      "tbz x14, #4, 240f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v25.8h }, [x22], #0x10\n"
+      "tbz x14, #3, 236f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
+      "st1 { v26.8h }, [x22], #0x10\n"
+      "tbz x14, #2, 234f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "tbz x14, #1, 233f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
+      "st1 { v27.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v11.h }[6], [x13]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
+      "st1 { v27.h }[6], [x22]\n"
       "b 248f\n"
       "233:"  // Height 5: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 248f\n"
-      "st1 { v11.h }[4], [x12]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
-      "st1 { v27.h }[4], [x21]\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v11.h }[4], [x13]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
+      "st1 { v27.h }[4], [x22]\n"
       "b 248f\n"
       "234:"  // Height 5: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 235f\n"
-      "str s11, [x12], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
-      "str s27, [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v11.h }[2], [x12]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
-      "st1 { v27.h }[2], [x21]\n"
+      "tbz x14, #1, 235f\n"
+      "str s11, [x13], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
+      "str s27, [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v11.h }[2], [x13]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
+      "st1 { v27.h }[2], [x22]\n"
       "b 248f\n"
       "235:"  // Height 5: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 248f\n"
-      "str h11, [x12, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
-      "str h27, [x21, #0x0]\n"
+      "tbz x14, #0, 248f\n"
+      "str h11, [x13, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
+      "str h27, [x22, #0x0]\n"
       "b 248f\n"
       "236:"  // Height 5: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 238f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "tbz x13, #1, 237f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
-      "st1 { v26.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v10.h }[6], [x12]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
-      "st1 { v26.h }[6], [x21]\n"
+      "tbz x14, #2, 238f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "tbz x14, #1, 237f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
+      "st1 { v26.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v10.h }[6], [x13]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
+      "st1 { v26.h }[6], [x22]\n"
       "b 248f\n"
       "237:"  // Height 5: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 248f\n"
-      "st1 { v10.h }[4], [x12]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
-      "st1 { v26.h }[4], [x21]\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v10.h }[4], [x13]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
+      "st1 { v26.h }[4], [x22]\n"
       "b 248f\n"
       "238:"  // Height 5: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 239f\n"
-      "str s10, [x12], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
-      "str s26, [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v10.h }[2], [x12]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
-      "st1 { v26.h }[2], [x21]\n"
+      "tbz x14, #1, 239f\n"
+      "str s10, [x13], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
+      "str s26, [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v10.h }[2], [x13]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
+      "st1 { v26.h }[2], [x22]\n"
       "b 248f\n"
       "239:"  // Height 5: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 248f\n"
-      "str h10, [x12, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
-      "str h26, [x21, #0x0]\n"
+      "tbz x14, #0, 248f\n"
+      "str h10, [x13, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
+      "str h26, [x22, #0x0]\n"
       "b 248f\n"
       "240:"  // Height 5: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 244f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "tbz x13, #2, 242f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "tbz x13, #1, 241f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
-      "st1 { v25.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v9.h }[6], [x12]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
-      "st1 { v25.h }[6], [x21]\n"
+      "tbz x14, #3, 244f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "tbz x14, #2, 242f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "tbz x14, #1, 241f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
+      "st1 { v25.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v9.h }[6], [x13]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
+      "st1 { v25.h }[6], [x22]\n"
       "b 248f\n"
       "241:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 248f\n"
-      "st1 { v9.h }[4], [x12]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
-      "st1 { v25.h }[4], [x21]\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v9.h }[4], [x13]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
+      "st1 { v25.h }[4], [x22]\n"
       "b 248f\n"
       "242:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 243f\n"
-      "str s9, [x12], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
-      "str s25, [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v9.h }[2], [x12]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
-      "st1 { v25.h }[2], [x21]\n"
+      "tbz x14, #1, 243f\n"
+      "str s9, [x13], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
+      "str s25, [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v9.h }[2], [x13]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
+      "st1 { v25.h }[2], [x22]\n"
       "b 248f\n"
       "243:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 248f\n"
-      "str h9, [x12, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
-      "str h25, [x21, #0x0]\n"
+      "tbz x14, #0, 248f\n"
+      "str h9, [x13, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
+      "str h25, [x22, #0x0]\n"
       "b 248f\n"
       "244:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 246f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "tbz x13, #1, 245f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
-      "st1 { v24.s }[2], [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v8.h }[6], [x12]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
-      "st1 { v24.h }[6], [x21]\n"
+      "tbz x14, #2, 246f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "tbz x14, #1, 245f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
+      "st1 { v24.s }[2], [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v8.h }[6], [x13]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
+      "st1 { v24.h }[6], [x22]\n"
       "b 248f\n"
       "245:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 248f\n"
-      "st1 { v8.h }[4], [x12]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
-      "st1 { v24.h }[4], [x21]\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v8.h }[4], [x13]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
+      "st1 { v24.h }[4], [x22]\n"
       "b 248f\n"
       "246:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 247f\n"
-      "str s8, [x12], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
-      "str s24, [x21], #0x4\n"
-      "tbz x13, #0, 248f\n"
-      "st1 { v8.h }[2], [x12]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
-      "st1 { v24.h }[2], [x21]\n"
+      "tbz x14, #1, 247f\n"
+      "str s8, [x13], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
+      "str s24, [x22], #0x4\n"
+      "tbz x14, #0, 248f\n"
+      "st1 { v8.h }[2], [x13]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
+      "st1 { v24.h }[2], [x22]\n"
       "b 248f\n"
       "247:"  // Height 5: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
-      "str h24, [x21, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
+      "str h24, [x22, #0x0]\n"
       "248:"  // Height 5: Partial direct writeback: Done
       "b 250f\n"
       "249:"  // Height 5: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "250:"  // Height 5: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 202b\n"
       "b 302f\n"
       "251:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0xc\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0xc\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "252:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x18\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x18\n"
       "bgt 253f\n"
-      "cmp x13, #0x10\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x9, x12\n"
       "bgt 253f\n"
-      "cmp x13, #0x8\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x10, x12\n"
       "bgt 253f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "253:"  // Height 6: B setup done
-      "cbz x14, 254f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 254f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -4162,310 +4162,310 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "b 273f\n"
       "254:"  // Height 6: no bias
       "tbz %x[flags], #0, 272f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "cmp x13, #0x20\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "cmp x14, #0x20\n"
+      "add x21, x22, x20, LSL #1\n"
       "bge 271f\n"
-      "tbz x13, #4, 262f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v28.8h }, [x20], #0x10\n"
-      "ld1 { v9.8h }, [x12], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
-      "ld1 { v25.8h }, [x21], #0x10\n"
-      "ld1 { v29.8h }, [x20], #0x10\n"
-      "tbz x13, #3, 258f\n"
-      "ld1 { v10.8h }, [x12], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
-      "ld1 { v26.8h }, [x21], #0x10\n"
-      "ld1 { v30.8h }, [x20], #0x10\n"
-      "tbz x13, #2, 256f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
-      "tbz x13, #1, 255f\n"
-      "ld1 { v11.s }[2], [x12], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
-      "ld1 { v27.s }[2], [x21], #0x4\n"
-      "ld1 { v31.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v11.h }[6], [x12]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
-      "ld1 { v27.h }[6], [x21]\n"
-      "ld1 { v31.h }[6], [x20]\n"
+      "tbz x14, #4, 262f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v28.8h }, [x21], #0x10\n"
+      "ld1 { v9.8h }, [x13], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
+      "ld1 { v25.8h }, [x22], #0x10\n"
+      "ld1 { v29.8h }, [x21], #0x10\n"
+      "tbz x14, #3, 258f\n"
+      "ld1 { v10.8h }, [x13], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
+      "ld1 { v26.8h }, [x22], #0x10\n"
+      "ld1 { v30.8h }, [x21], #0x10\n"
+      "tbz x14, #2, 256f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
+      "tbz x14, #1, 255f\n"
+      "ld1 { v11.s }[2], [x13], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
+      "ld1 { v27.s }[2], [x22], #0x4\n"
+      "ld1 { v31.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v11.h }[6], [x13]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
+      "ld1 { v27.h }[6], [x22]\n"
+      "ld1 { v31.h }[6], [x21]\n"
       "b 270f\n"
       "255:"  // Height 6: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v11.h }[4], [x12]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
-      "ld1 { v27.h }[4], [x21]\n"
-      "ld1 { v31.h }[4], [x20]\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v11.h }[4], [x13]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
+      "ld1 { v27.h }[4], [x22]\n"
+      "ld1 { v31.h }[4], [x21]\n"
       "b 270f\n"
       "256:"  // Height 6: Partial accumulate: partial_2_24
-      "tbz x13, #1, 257f\n"
-      "ldr s11, [x12], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
-      "ldr s27, [x21], #0x4\n"
-      "ldr s31, [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v11.h }[2], [x12]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
-      "ld1 { v27.h }[2], [x21]\n"
-      "ld1 { v31.h }[2], [x20]\n"
+      "tbz x14, #1, 257f\n"
+      "ldr s11, [x13], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
+      "ldr s27, [x22], #0x4\n"
+      "ldr s31, [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v11.h }[2], [x13]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
+      "ld1 { v27.h }[2], [x22]\n"
+      "ld1 { v31.h }[2], [x21]\n"
       "b 270f\n"
       "257:"  // Height 6: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 270f\n"
-      "ldr h11, [x12, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
-      "ldr h27, [x21, #0x0]\n"
-      "ldr h31, [x20, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 270f\n"
+      "ldr h11, [x13, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
+      "ldr h27, [x22, #0x0]\n"
+      "ldr h31, [x21, #0x0]\n"
       "b 270f\n"
       "258:"  // Height 6: Partial accumulate: partial_4_16
-      "tbz x13, #2, 260f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
-      "tbz x13, #1, 259f\n"
-      "ld1 { v10.s }[2], [x12], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
-      "ld1 { v26.s }[2], [x21], #0x4\n"
-      "ld1 { v30.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v10.h }[6], [x12]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
-      "ld1 { v26.h }[6], [x21]\n"
-      "ld1 { v30.h }[6], [x20]\n"
+      "tbz x14, #2, 260f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
+      "tbz x14, #1, 259f\n"
+      "ld1 { v10.s }[2], [x13], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
+      "ld1 { v26.s }[2], [x22], #0x4\n"
+      "ld1 { v30.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v10.h }[6], [x13]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
+      "ld1 { v26.h }[6], [x22]\n"
+      "ld1 { v30.h }[6], [x21]\n"
       "b 270f\n"
       "259:"  // Height 6: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v10.h }[4], [x12]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
-      "ld1 { v26.h }[4], [x21]\n"
-      "ld1 { v30.h }[4], [x20]\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v10.h }[4], [x13]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
+      "ld1 { v26.h }[4], [x22]\n"
+      "ld1 { v30.h }[4], [x21]\n"
       "b 270f\n"
       "260:"  // Height 6: Partial accumulate: partial_2_16
-      "tbz x13, #1, 261f\n"
-      "ldr s10, [x12], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
-      "ldr s26, [x21], #0x4\n"
-      "ldr s30, [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v10.h }[2], [x12]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
-      "ld1 { v26.h }[2], [x21]\n"
-      "ld1 { v30.h }[2], [x20]\n"
+      "tbz x14, #1, 261f\n"
+      "ldr s10, [x13], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
+      "ldr s26, [x22], #0x4\n"
+      "ldr s30, [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v10.h }[2], [x13]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
+      "ld1 { v26.h }[2], [x22]\n"
+      "ld1 { v30.h }[2], [x21]\n"
       "b 270f\n"
       "261:"  // Height 6: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 270f\n"
-      "ldr h10, [x12, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
-      "ldr h26, [x21, #0x0]\n"
-      "ldr h30, [x20, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 270f\n"
+      "ldr h10, [x13, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
+      "ldr h26, [x22, #0x0]\n"
+      "ldr h30, [x21, #0x0]\n"
       "b 270f\n"
       "262:"  // Height 6: Partial accumulate: partial_8_0
-      "tbz x13, #3, 266f\n"
-      "ld1 { v8.8h }, [x12], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v28.8h }, [x20], #0x10\n"
-      "tbz x13, #2, 264f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
-      "tbz x13, #1, 263f\n"
-      "ld1 { v9.s }[2], [x12], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
-      "ld1 { v25.s }[2], [x21], #0x4\n"
-      "ld1 { v29.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v9.h }[6], [x12]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
-      "ld1 { v25.h }[6], [x21]\n"
-      "ld1 { v29.h }[6], [x20]\n"
+      "tbz x14, #3, 266f\n"
+      "ld1 { v8.8h }, [x13], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v28.8h }, [x21], #0x10\n"
+      "tbz x14, #2, 264f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
+      "tbz x14, #1, 263f\n"
+      "ld1 { v9.s }[2], [x13], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
+      "ld1 { v25.s }[2], [x22], #0x4\n"
+      "ld1 { v29.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v9.h }[6], [x13]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
+      "ld1 { v25.h }[6], [x22]\n"
+      "ld1 { v29.h }[6], [x21]\n"
       "b 270f\n"
       "263:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v9.h }[4], [x12]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
-      "ld1 { v25.h }[4], [x21]\n"
-      "ld1 { v29.h }[4], [x20]\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v9.h }[4], [x13]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
+      "ld1 { v25.h }[4], [x22]\n"
+      "ld1 { v29.h }[4], [x21]\n"
       "b 270f\n"
       "264:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x13, #1, 265f\n"
-      "ldr s9, [x12], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
-      "ldr s25, [x21], #0x4\n"
-      "ldr s29, [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v9.h }[2], [x12]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
-      "ld1 { v25.h }[2], [x21]\n"
-      "ld1 { v29.h }[2], [x20]\n"
+      "tbz x14, #1, 265f\n"
+      "ldr s9, [x13], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
+      "ldr s25, [x22], #0x4\n"
+      "ldr s29, [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v9.h }[2], [x13]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
+      "ld1 { v25.h }[2], [x22]\n"
+      "ld1 { v29.h }[2], [x21]\n"
       "b 270f\n"
       "265:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 270f\n"
-      "ldr h9, [x12, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
-      "ldr h25, [x21, #0x0]\n"
-      "ldr h29, [x20, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 270f\n"
+      "ldr h9, [x13, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
+      "ldr h25, [x22, #0x0]\n"
+      "ldr h29, [x21, #0x0]\n"
       "b 270f\n"
       "266:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x13, #2, 268f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
-      "tbz x13, #1, 267f\n"
-      "ld1 { v8.s }[2], [x12], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
-      "ld1 { v24.s }[2], [x21], #0x4\n"
-      "ld1 { v28.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v8.h }[6], [x12]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
-      "ld1 { v24.h }[6], [x21]\n"
-      "ld1 { v28.h }[6], [x20]\n"
+      "tbz x14, #2, 268f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
+      "tbz x14, #1, 267f\n"
+      "ld1 { v8.s }[2], [x13], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
+      "ld1 { v24.s }[2], [x22], #0x4\n"
+      "ld1 { v28.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v8.h }[6], [x13]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
+      "ld1 { v24.h }[6], [x22]\n"
+      "ld1 { v28.h }[6], [x21]\n"
       "b 270f\n"
       "267:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v8.h }[4], [x12]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
-      "ld1 { v24.h }[4], [x21]\n"
-      "ld1 { v28.h }[4], [x20]\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v8.h }[4], [x13]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
+      "ld1 { v24.h }[4], [x22]\n"
+      "ld1 { v28.h }[4], [x21]\n"
       "b 270f\n"
       "268:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x13, #1, 269f\n"
-      "ldr s8, [x12], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
-      "ldr s24, [x21], #0x4\n"
-      "ldr s28, [x20], #0x4\n"
-      "tbz x13, #0, 270f\n"
-      "ld1 { v8.h }[2], [x12]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
-      "ld1 { v24.h }[2], [x21]\n"
-      "ld1 { v28.h }[2], [x20]\n"
+      "tbz x14, #1, 269f\n"
+      "ldr s8, [x13], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
+      "ldr s24, [x22], #0x4\n"
+      "ldr s28, [x21], #0x4\n"
+      "tbz x14, #0, 270f\n"
+      "ld1 { v8.h }[2], [x13]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
+      "ld1 { v24.h }[2], [x22]\n"
+      "ld1 { v28.h }[2], [x21]\n"
       "b 270f\n"
       "269:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr h8, [x12, #0x0]\n"
-      "ldr h12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
-      "ldr h24, [x21, #0x0]\n"
-      "ldr h28, [x20, #0x0]\n"
+      "ldr h8, [x13, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
+      "ldr h24, [x22, #0x0]\n"
+      "ldr h28, [x21, #0x0]\n"
       "270:"  // Height 6: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 273f\n"
       "271:"  // Height 6: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "b 273f\n"
       "272:"  // Height 6: no accumulate
       "movi v8.16b, #0x0\n"
@@ -4493,515 +4493,515 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "273:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "274:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 275f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 276f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 276f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 276f\n"
       "275:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "276:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 279f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 278f\n"
       "277:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
       "fmla v28.8h, v6.8h, v5.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
       "fmla v29.8h, v7.8h, v5.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
       "fmla v30.8h, v6.8h, v5.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
       "fmla v31.8h, v7.8h, v5.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
       "fmla v28.8h, v6.8h, v5.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
       "fmla v29.8h, v7.8h, v5.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
       "fmla v30.8h, v6.8h, v5.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
       "fmla v31.8h, v7.8h, v5.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
       "fmla v28.8h, v6.8h, v5.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
       "fmla v29.8h, v7.8h, v5.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
       "fmla v30.8h, v6.8h, v5.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
       "fmla v31.8h, v7.8h, v5.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
       "fmla v28.8h, v6.8h, v5.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
       "fmla v29.8h, v7.8h, v5.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
       "fmla v30.8h, v6.8h, v5.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
       "fmla v31.8h, v7.8h, v5.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
       "fmla v28.8h, v6.8h, v5.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
       "fmla v29.8h, v7.8h, v5.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
       "fmla v30.8h, v6.8h, v5.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
       "fmla v31.8h, v7.8h, v5.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
       "fmla v28.8h, v6.8h, v5.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
       "fmla v29.8h, v7.8h, v5.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
       "fmla v30.8h, v6.8h, v5.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
       "fmla v31.8h, v7.8h, v5.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
       "fmla v28.8h, v6.8h, v5.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
       "fmla v29.8h, v7.8h, v5.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "fmla v26.8h, v6.8h, v4.h[7]\n"
       "fmla v30.8h, v6.8h, v5.h[7]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 277b\n"
       "278:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x26, x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
+      "add x22, x22, #0x10\n"
       "add x21, x21, #0x10\n"
-      "add x20, x20, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
       "fmla v28.8h, v6.8h, v5.h[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
       "fmla v29.8h, v7.8h, v5.h[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
       "fmla v30.8h, v6.8h, v5.h[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
       "fmla v31.8h, v7.8h, v5.h[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
       "fmla v28.8h, v6.8h, v5.h[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
       "fmla v29.8h, v7.8h, v5.h[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
       "fmla v30.8h, v6.8h, v5.h[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
       "fmla v31.8h, v7.8h, v5.h[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
       "fmla v28.8h, v6.8h, v5.h[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
       "fmla v29.8h, v7.8h, v5.h[3]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
       "fmla v30.8h, v6.8h, v5.h[3]\n"
-      "ldr q6, [x11, #0x40]\n"
+      "ldr q6, [x12, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
       "fmla v31.8h, v7.8h, v5.h[3]\n"
-      "ldr q7, [x10, #0x40]\n"
+      "ldr q7, [x11, #0x40]\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
       "fmla v28.8h, v6.8h, v5.h[4]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
       "fmla v29.8h, v7.8h, v5.h[4]\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
       "fmla v30.8h, v6.8h, v5.h[4]\n"
-      "ldr q6, [x11, #0x50]\n"
+      "ldr q6, [x12, #0x50]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
       "fmla v31.8h, v7.8h, v5.h[4]\n"
-      "ldr q7, [x10, #0x50]\n"
+      "ldr q7, [x11, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
       "fmla v28.8h, v6.8h, v5.h[5]\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
       "fmla v29.8h, v7.8h, v5.h[5]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
       "fmla v30.8h, v6.8h, v5.h[5]\n"
-      "ldr q6, [x11, #0x60]\n"
+      "ldr q6, [x12, #0x60]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
       "fmla v31.8h, v7.8h, v5.h[5]\n"
-      "ldr q7, [x10, #0x60]\n"
+      "ldr q7, [x11, #0x60]\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
       "fmla v28.8h, v6.8h, v5.h[6]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
       "fmla v29.8h, v7.8h, v5.h[6]\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
       "fmla v30.8h, v6.8h, v5.h[6]\n"
-      "ldr q6, [x11, #0x70]\n"
-      "add x11, x11, #0x80\n"
+      "ldr q6, [x12, #0x70]\n"
+      "add x12, x12, #0x80\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
       "fmla v31.8h, v7.8h, v5.h[6]\n"
-      "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "ldr q7, [x11, #0x70]\n"
+      "add x11, x11, #0x80\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
       "fmla v28.8h, v6.8h, v5.h[7]\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
       "fmla v29.8h, v7.8h, v5.h[7]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
@@ -5015,35 +5015,35 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
       "279:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 281f\n"
+      "cbz x27, 281f\n"
       "280:"  // Height 6: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
-      "add x10, x10, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x10, x10, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
@@ -5056,23 +5056,23 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
-      "cbnz x26, 280b\n"
+      "cbnz x27, 280b\n"
       "281:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 274b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "tbz %x[flags], #1, 282f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v1.8h\n"
       "fmin v9.8h, v9.8h, v1.8h\n"
       "fmin v10.8h, v10.8h, v1.8h\n"
@@ -5122,306 +5122,306 @@ void a64_ffhybrid_fp16_mla_6x32 (
       "fmax v30.8h, v30.8h, v0.8h\n"
       "fmax v31.8h, v31.8h, v0.8h\n"
       "282:"  // Height 6: No activation
-      "cmp x13, #0x20\n"
+      "cmp x14, #0x20\n"
       "bge 299f\n"
-      "tbz x13, #4, 290f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v9.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v25.8h }, [x21], #0x10\n"
-      "st1 { v28.8h }, [x20], #0x10\n"
-      "st1 { v29.8h }, [x20], #0x10\n"
-      "tbz x13, #3, 286f\n"
-      "st1 { v10.8h }, [x12], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
-      "st1 { v26.8h }, [x21], #0x10\n"
-      "st1 { v30.8h }, [x20], #0x10\n"
-      "tbz x13, #2, 284f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
-      "tbz x13, #1, 283f\n"
-      "st1 { v11.s }[2], [x12], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
-      "st1 { v27.s }[2], [x21], #0x4\n"
-      "st1 { v31.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v11.h }[6], [x12]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
-      "st1 { v27.h }[6], [x21]\n"
-      "st1 { v31.h }[6], [x20]\n"
+      "tbz x14, #4, 290f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v9.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v25.8h }, [x22], #0x10\n"
+      "st1 { v28.8h }, [x21], #0x10\n"
+      "st1 { v29.8h }, [x21], #0x10\n"
+      "tbz x14, #3, 286f\n"
+      "st1 { v10.8h }, [x13], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
+      "st1 { v26.8h }, [x22], #0x10\n"
+      "st1 { v30.8h }, [x21], #0x10\n"
+      "tbz x14, #2, 284f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
+      "tbz x14, #1, 283f\n"
+      "st1 { v11.s }[2], [x13], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
+      "st1 { v27.s }[2], [x22], #0x4\n"
+      "st1 { v31.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v11.h }[6], [x13]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
+      "st1 { v27.h }[6], [x22]\n"
+      "st1 { v31.h }[6], [x21]\n"
       "b 298f\n"
       "283:"  // Height 6: Partial direct writeback: partial_1_28
-      "tbz x13, #0, 298f\n"
-      "st1 { v11.h }[4], [x12]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
-      "st1 { v27.h }[4], [x21]\n"
-      "st1 { v31.h }[4], [x20]\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v11.h }[4], [x13]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
+      "st1 { v27.h }[4], [x22]\n"
+      "st1 { v31.h }[4], [x21]\n"
       "b 298f\n"
       "284:"  // Height 6: Partial direct writeback: partial_2_24
-      "tbz x13, #1, 285f\n"
-      "str s11, [x12], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
-      "str s27, [x21], #0x4\n"
-      "str s31, [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v11.h }[2], [x12]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
-      "st1 { v27.h }[2], [x21]\n"
-      "st1 { v31.h }[2], [x20]\n"
+      "tbz x14, #1, 285f\n"
+      "str s11, [x13], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
+      "str s27, [x22], #0x4\n"
+      "str s31, [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v11.h }[2], [x13]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
+      "st1 { v27.h }[2], [x22]\n"
+      "st1 { v31.h }[2], [x21]\n"
       "b 298f\n"
       "285:"  // Height 6: Partial direct writeback: partial_1_24
-      "tbz x13, #0, 298f\n"
-      "str h11, [x12, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
-      "str h27, [x21, #0x0]\n"
-      "str h31, [x20, #0x0]\n"
+      "tbz x14, #0, 298f\n"
+      "str h11, [x13, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
+      "str h27, [x22, #0x0]\n"
+      "str h31, [x21, #0x0]\n"
       "b 298f\n"
       "286:"  // Height 6: Partial direct writeback: partial_4_16
-      "tbz x13, #2, 288f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
-      "tbz x13, #1, 287f\n"
-      "st1 { v10.s }[2], [x12], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
-      "st1 { v26.s }[2], [x21], #0x4\n"
-      "st1 { v30.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v10.h }[6], [x12]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
-      "st1 { v26.h }[6], [x21]\n"
-      "st1 { v30.h }[6], [x20]\n"
+      "tbz x14, #2, 288f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
+      "tbz x14, #1, 287f\n"
+      "st1 { v10.s }[2], [x13], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
+      "st1 { v26.s }[2], [x22], #0x4\n"
+      "st1 { v30.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v10.h }[6], [x13]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
+      "st1 { v26.h }[6], [x22]\n"
+      "st1 { v30.h }[6], [x21]\n"
       "b 298f\n"
       "287:"  // Height 6: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 298f\n"
-      "st1 { v10.h }[4], [x12]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
-      "st1 { v26.h }[4], [x21]\n"
-      "st1 { v30.h }[4], [x20]\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v10.h }[4], [x13]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
+      "st1 { v26.h }[4], [x22]\n"
+      "st1 { v30.h }[4], [x21]\n"
       "b 298f\n"
       "288:"  // Height 6: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 289f\n"
-      "str s10, [x12], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
-      "str s26, [x21], #0x4\n"
-      "str s30, [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v10.h }[2], [x12]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
-      "st1 { v26.h }[2], [x21]\n"
-      "st1 { v30.h }[2], [x20]\n"
+      "tbz x14, #1, 289f\n"
+      "str s10, [x13], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
+      "str s26, [x22], #0x4\n"
+      "str s30, [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v10.h }[2], [x13]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
+      "st1 { v26.h }[2], [x22]\n"
+      "st1 { v30.h }[2], [x21]\n"
       "b 298f\n"
       "289:"  // Height 6: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 298f\n"
-      "str h10, [x12, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
-      "str h26, [x21, #0x0]\n"
-      "str h30, [x20, #0x0]\n"
+      "tbz x14, #0, 298f\n"
+      "str h10, [x13, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
+      "str h26, [x22, #0x0]\n"
+      "str h30, [x21, #0x0]\n"
       "b 298f\n"
       "290:"  // Height 6: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 294f\n"
-      "st1 { v8.8h }, [x12], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v28.8h }, [x20], #0x10\n"
-      "tbz x13, #2, 292f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
-      "tbz x13, #1, 291f\n"
-      "st1 { v9.s }[2], [x12], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
-      "st1 { v25.s }[2], [x21], #0x4\n"
-      "st1 { v29.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v9.h }[6], [x12]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
-      "st1 { v25.h }[6], [x21]\n"
-      "st1 { v29.h }[6], [x20]\n"
+      "tbz x14, #3, 294f\n"
+      "st1 { v8.8h }, [x13], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v28.8h }, [x21], #0x10\n"
+      "tbz x14, #2, 292f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
+      "tbz x14, #1, 291f\n"
+      "st1 { v9.s }[2], [x13], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
+      "st1 { v25.s }[2], [x22], #0x4\n"
+      "st1 { v29.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v9.h }[6], [x13]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
+      "st1 { v25.h }[6], [x22]\n"
+      "st1 { v29.h }[6], [x21]\n"
       "b 298f\n"
       "291:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 298f\n"
-      "st1 { v9.h }[4], [x12]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
-      "st1 { v25.h }[4], [x21]\n"
-      "st1 { v29.h }[4], [x20]\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v9.h }[4], [x13]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
+      "st1 { v25.h }[4], [x22]\n"
+      "st1 { v29.h }[4], [x21]\n"
       "b 298f\n"
       "292:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 293f\n"
-      "str s9, [x12], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
-      "str s25, [x21], #0x4\n"
-      "str s29, [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v9.h }[2], [x12]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
-      "st1 { v25.h }[2], [x21]\n"
-      "st1 { v29.h }[2], [x20]\n"
+      "tbz x14, #1, 293f\n"
+      "str s9, [x13], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
+      "str s25, [x22], #0x4\n"
+      "str s29, [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v9.h }[2], [x13]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
+      "st1 { v25.h }[2], [x22]\n"
+      "st1 { v29.h }[2], [x21]\n"
       "b 298f\n"
       "293:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 298f\n"
-      "str h9, [x12, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
-      "str h25, [x21, #0x0]\n"
-      "str h29, [x20, #0x0]\n"
+      "tbz x14, #0, 298f\n"
+      "str h9, [x13, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
+      "str h25, [x22, #0x0]\n"
+      "str h29, [x21, #0x0]\n"
       "b 298f\n"
       "294:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 296f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
-      "tbz x13, #1, 295f\n"
-      "st1 { v8.s }[2], [x12], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
-      "st1 { v24.s }[2], [x21], #0x4\n"
-      "st1 { v28.s }[2], [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v8.h }[6], [x12]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
-      "st1 { v24.h }[6], [x21]\n"
-      "st1 { v28.h }[6], [x20]\n"
+      "tbz x14, #2, 296f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
+      "tbz x14, #1, 295f\n"
+      "st1 { v8.s }[2], [x13], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
+      "st1 { v24.s }[2], [x22], #0x4\n"
+      "st1 { v28.s }[2], [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v8.h }[6], [x13]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
+      "st1 { v24.h }[6], [x22]\n"
+      "st1 { v28.h }[6], [x21]\n"
       "b 298f\n"
       "295:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 298f\n"
-      "st1 { v8.h }[4], [x12]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
-      "st1 { v24.h }[4], [x21]\n"
-      "st1 { v28.h }[4], [x20]\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v8.h }[4], [x13]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
+      "st1 { v24.h }[4], [x22]\n"
+      "st1 { v28.h }[4], [x21]\n"
       "b 298f\n"
       "296:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 297f\n"
-      "str s8, [x12], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
-      "str s24, [x21], #0x4\n"
-      "str s28, [x20], #0x4\n"
-      "tbz x13, #0, 298f\n"
-      "st1 { v8.h }[2], [x12]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
-      "st1 { v24.h }[2], [x21]\n"
-      "st1 { v28.h }[2], [x20]\n"
+      "tbz x14, #1, 297f\n"
+      "str s8, [x13], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
+      "str s24, [x22], #0x4\n"
+      "str s28, [x21], #0x4\n"
+      "tbz x14, #0, 298f\n"
+      "st1 { v8.h }[2], [x13]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
+      "st1 { v24.h }[2], [x22]\n"
+      "st1 { v28.h }[2], [x21]\n"
       "b 298f\n"
       "297:"  // Height 6: Partial direct writeback: partial_1_0
-      "str h8, [x12, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
-      "str h24, [x21, #0x0]\n"
-      "str h28, [x20, #0x0]\n"
+      "str h8, [x13, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
+      "str h24, [x22, #0x0]\n"
+      "str h28, [x21, #0x0]\n"
       "298:"  // Height 6: Partial direct writeback: Done
       "b 300f\n"
       "299:"  // Height 6: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
-      "str q28, [x20, #0x0]\n"
-      "str q29, [x20, #0x10]\n"
-      "str q30, [x20, #0x20]\n"
-      "str q31, [x20, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
+      "str q28, [x21, #0x0]\n"
+      "str q29, [x21, #0x10]\n"
+      "str q30, [x21, #0x20]\n"
+      "str q31, [x21, #0x30]\n"
       "300:"  // Height 6: Writeback done
-      "subs x13, x13, #0x20\n"
+      "subs x14, x14, #0x20\n"
       "bgt 252b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 302f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 301f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "301:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "302:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp
index f811116a06..e0fbe17bad 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -104,100 +104,100 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 69f\n"
       "beq 35f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 3f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "cbz x14, 4f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
-      "add x14, x14, #0x40\n"
+      "cbz x15, 4f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
+      "add x15, x15, #0x40\n"
       "b 15f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 14f\n"
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 13f\n"
-      "tbz x13, #3, 8f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 6f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 5f\n"
-      "ldr d11, [x12], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v11.s }[2], [x12]\n"
+      "tbz x14, #3, 8f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 6f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 5f\n"
+      "ldr d11, [x13], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v11.s }[2], [x13]\n"
       "b 12f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s11, [x12, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s11, [x13, #0x0]\n"
       "b 12f\n"
       "6:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x13, #1, 7f\n"
-      "ldr d10, [x12], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v10.s }[2], [x12]\n"
+      "tbz x14, #1, 7f\n"
+      "ldr d10, [x13], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v10.s }[2], [x13]\n"
       "b 12f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s10, [x12, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s10, [x13, #0x0]\n"
       "b 12f\n"
       "8:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x13, #2, 10f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 9f\n"
-      "ldr d9, [x12], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v9.s }[2], [x12]\n"
+      "tbz x14, #2, 10f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 9f\n"
+      "ldr d9, [x13], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v9.s }[2], [x13]\n"
       "b 12f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 12f\n"
-      "ldr s9, [x12, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 12f\n"
+      "ldr s9, [x13, #0x0]\n"
       "b 12f\n"
       "10:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x13, #1, 11f\n"
-      "ldr d8, [x12], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 12f\n"
-      "ld1 { v8.s }[2], [x12]\n"
+      "tbz x14, #1, 11f\n"
+      "ldr d8, [x13], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 12f\n"
+      "ld1 { v8.s }[2], [x13]\n"
       "b 12f\n"
       "11:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x13, #0x0]\n"
+      "mov x20, #0x0\n"
       "12:"  // Height 1: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 15f\n"
       "13:"  // Height 1: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
       "b 15f\n"
       "14:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -205,136 +205,136 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "15:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 18f\n"
       "17:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "18:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 21f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 20f\n"
       "19:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x8\n"
+      "ldr q7, [x9, #0x30]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "add x25, x25, #0x10\n"
-      "ldr q0, [x25, #0x0]\n"
+      "add x26, x26, #0x10\n"
+      "ldr q0, [x26, #0x0]\n"
+      "add x12, x12, #0x40\n"
+      "ldr q6, [x12, #0x0]\n"
       "add x11, x11, #0x40\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "add x10, x10, #0x40\n"
-      "ldr q7, [x10, #0x0]\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "bge 19b\n"
       "20:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q7, [x11, #0x30]\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "sub x26, x26, #0x4\n"
+      "ldr q7, [x9, #0x30]\n"
+      "sub x27, x27, #0x4\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x12, x12, #0x40\n"
       "add x11, x11, #0x40\n"
       "add x10, x10, #0x40\n"
       "add x9, x9, #0x40\n"
-      "add x28, x28, #0x40\n"
       "21:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 23f\n"
+      "cbz x27, 23f\n"
       "22:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "sub x26, x26, #0x1\n"
-      "ldr q7, [x10, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "sub x27, x27, #0x1\n"
+      "ldr q7, [x11, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
       "add x9, x9, #0x10\n"
-      "add x28, x28, #0x10\n"
-      "cbnz x26, 22b\n"
+      "cbnz x27, 22b\n"
       "23:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
       "tbz %x[flags], #1, 24f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -344,185 +344,185 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v10.4s, v10.4s, v0.4s\n"
       "fmax v11.4s, v11.4s, v0.4s\n"
       "24:"  // Height 1: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 33f\n"
-      "tbz x13, #3, 28f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 26f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 25f\n"
-      "str d11, [x12], #0x8\n"
-      "tbz x13, #0, 32f\n"
-      "st1 { v11.s }[2], [x12]\n"
+      "tbz x14, #3, 28f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 26f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 25f\n"
+      "str d11, [x13], #0x8\n"
+      "tbz x14, #0, 32f\n"
+      "st1 { v11.s }[2], [x13]\n"
       "b 32f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 32f\n"
-      "str s11, [x12, #0x0]\n"
+      "tbz x14, #0, 32f\n"
+      "str s11, [x13, #0x0]\n"
       "b 32f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 27f\n"
-      "str d10, [x12], #0x8\n"
-      "tbz x13, #0, 32f\n"
-      "st1 { v10.s }[2], [x12]\n"
+      "tbz x14, #1, 27f\n"
+      "str d10, [x13], #0x8\n"
+      "tbz x14, #0, 32f\n"
+      "st1 { v10.s }[2], [x13]\n"
       "b 32f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 32f\n"
-      "str s10, [x12, #0x0]\n"
+      "tbz x14, #0, 32f\n"
+      "str s10, [x13, #0x0]\n"
       "b 32f\n"
       "28:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 30f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 29f\n"
-      "str d9, [x12], #0x8\n"
-      "tbz x13, #0, 32f\n"
-      "st1 { v9.s }[2], [x12]\n"
+      "tbz x14, #2, 30f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 29f\n"
+      "str d9, [x13], #0x8\n"
+      "tbz x14, #0, 32f\n"
+      "st1 { v9.s }[2], [x13]\n"
       "b 32f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 32f\n"
-      "str s9, [x12, #0x0]\n"
+      "tbz x14, #0, 32f\n"
+      "str s9, [x13, #0x0]\n"
       "b 32f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 31f\n"
-      "str d8, [x12], #0x8\n"
-      "tbz x13, #0, 32f\n"
-      "st1 { v8.s }[2], [x12]\n"
+      "tbz x14, #1, 31f\n"
+      "str d8, [x13], #0x8\n"
+      "tbz x14, #0, 32f\n"
+      "st1 { v8.s }[2], [x13]\n"
       "b 32f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
       "32:"  // Height 1: Partial direct writeback: Done
       "b 34f\n"
       "33:"  // Height 1: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
       "34:"  // Height 1: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 2b\n"
       "b 206f\n"
       "35:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "36:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 37f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 37f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 37f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "37:"  // Height 2: B setup done
-      "cbz x14, 38f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 38f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "b 49f\n"
       "38:"  // Height 2: no bias
       "tbz %x[flags], #0, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x13, #0x10\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x14, #0x10\n"
+      "add x25, x13, x20, LSL #2\n"
       "bge 47f\n"
-      "tbz x13, #3, 42f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "tbz x13, #2, 40f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 39f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 46f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "tbz x14, #3, 42f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "tbz x14, #2, 40f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 39f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 46f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 46f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 46f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 46f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 46f\n"
       "40:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x13, #1, 41f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 46f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "tbz x14, #1, 41f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 46f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 46f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 46f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 46f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 46f\n"
       "42:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x13, #2, 44f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 43f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 46f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "tbz x14, #2, 44f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 43f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 46f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 46f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 46f\n"
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 46f\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 46f\n"
       "44:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x13, #1, 45f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 46f\n"
-      "ld1 { v8.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "tbz x14, #1, 45f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 46f\n"
+      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 46f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "46:"  // Height 2: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 49f\n"
       "47:"  // Height 2: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "b 49f\n"
       "48:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -534,182 +534,182 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "49:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "50:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 51f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 52f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 52f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 52f\n"
       "51:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "52:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 55f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 54f\n"
       "53:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "sub x26, x26, #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x4\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "ldr q7, [x9, #0x0]\n"
+      "cmp x27, #0x8\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x12, #0x10]\n"
+      "add x26, x26, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x11, #0x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 53b\n"
       "54:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "sub x26, x26, #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x4\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x12, #0x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "55:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 57f\n"
+      "cbz x27, 57f\n"
       "56:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
-      "add x10, x10, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
+      "add x10, x10, #0x10\n"
       "add x9, x9, #0x10\n"
-      "add x28, x28, #0x10\n"
-      "cbnz x26, 56b\n"
+      "cbnz x27, 56b\n"
       "57:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 50b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
       "tbz %x[flags], #1, 58f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -727,230 +727,230 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v14.4s, v14.4s, v0.4s\n"
       "fmax v15.4s, v15.4s, v0.4s\n"
       "58:"  // Height 2: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 67f\n"
-      "tbz x13, #3, 62f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "tbz x13, #2, 60f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 59f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "tbz x13, #0, 66f\n"
-      "st1 { v11.s }[2], [x12]\n"
-      "st1 { v15.s }[2], [x24]\n"
+      "tbz x14, #3, 62f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "tbz x14, #2, 60f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 59f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "tbz x14, #0, 66f\n"
+      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v15.s }[2], [x25]\n"
       "b 66f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 66f\n"
-      "str s11, [x12, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
+      "tbz x14, #0, 66f\n"
+      "str s11, [x13, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
       "b 66f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 61f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "tbz x13, #0, 66f\n"
-      "st1 { v10.s }[2], [x12]\n"
-      "st1 { v14.s }[2], [x24]\n"
+      "tbz x14, #1, 61f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "tbz x14, #0, 66f\n"
+      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v14.s }[2], [x25]\n"
       "b 66f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 66f\n"
-      "str s10, [x12, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
+      "tbz x14, #0, 66f\n"
+      "str s10, [x13, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
       "b 66f\n"
       "62:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 64f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "tbz x13, #1, 63f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "tbz x13, #0, 66f\n"
-      "st1 { v9.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x24]\n"
+      "tbz x14, #2, 64f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "tbz x14, #1, 63f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "tbz x14, #0, 66f\n"
+      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x25]\n"
       "b 66f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 66f\n"
-      "str s9, [x12, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
+      "tbz x14, #0, 66f\n"
+      "str s9, [x13, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
       "b 66f\n"
       "64:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 65f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "tbz x13, #0, 66f\n"
-      "st1 { v8.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x24]\n"
+      "tbz x14, #1, 65f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "tbz x14, #0, 66f\n"
+      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x25]\n"
       "b 66f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
       "66:"  // Height 2: Partial direct writeback: Done
       "b 68f\n"
       "67:"  // Height 2: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
       "68:"  // Height 2: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 36b\n"
       "b 206f\n"
       "69:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "70:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 71f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 71f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 71f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "71:"  // Height 3: B setup done
-      "cbz x14, 72f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 72f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 83f\n"
       "72:"  // Height 3: no bias
       "tbz %x[flags], #0, 82f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 81f\n"
-      "tbz x13, #3, 76f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "tbz x13, #2, 74f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 73f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d19, [x23], #0x8\n"
-      "tbz x13, #0, 80f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "tbz x14, #3, 76f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "tbz x14, #2, 74f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 73f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "tbz x14, #0, 80f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 80f\n"
       "73:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 80f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 80f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 80f\n"
       "74:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x13, #1, 75f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "tbz x13, #0, 80f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "tbz x14, #1, 75f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "tbz x14, #0, 80f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 80f\n"
       "75:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 80f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 80f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 80f\n"
       "76:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x13, #2, 78f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 77f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d17, [x23], #0x8\n"
-      "tbz x13, #0, 80f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "tbz x14, #2, 78f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 77f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "tbz x14, #0, 80f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 80f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 80f\n"
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 80f\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
       "b 80f\n"
       "78:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x13, #1, 79f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "tbz x13, #0, 80f\n"
-      "ld1 { v8.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
+      "tbz x14, #1, 79f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "tbz x14, #0, 80f\n"
+      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
       "b 80f\n"
       "79:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s16, [x23, #0x0]\n"
+      "ldr s8, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
       "80:"  // Height 3: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 83f\n"
       "81:"  // Height 3: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
       "b 83f\n"
       "82:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -966,180 +966,180 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "83:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "84:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 85f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 86f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 86f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 86f\n"
       "85:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "86:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 89f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 88f\n"
       "87:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 87b\n"
       "88:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
@@ -1147,46 +1147,46 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "89:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 91f\n"
+      "cbz x27, 91f\n"
       "90:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x11, x11, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
-      "add x9, x9, #0x10\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "cbnz x26, 90b\n"
+      "cbnz x27, 90b\n"
       "91:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 84b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 92f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -1212,140 +1212,140 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v18.4s, v18.4s, v0.4s\n"
       "fmax v19.4s, v19.4s, v0.4s\n"
       "92:"  // Height 3: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 101f\n"
-      "tbz x13, #3, 96f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "tbz x13, #2, 94f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 93f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "tbz x13, #0, 100f\n"
-      "st1 { v11.s }[2], [x12]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "tbz x14, #3, 96f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "tbz x14, #2, 94f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 93f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "tbz x14, #0, 100f\n"
+      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 100f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 100f\n"
-      "str s11, [x12, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "tbz x14, #0, 100f\n"
+      "str s11, [x13, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 100f\n"
       "94:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 95f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "tbz x13, #0, 100f\n"
-      "st1 { v10.s }[2], [x12]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "tbz x14, #1, 95f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "tbz x14, #0, 100f\n"
+      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 100f\n"
       "95:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 100f\n"
-      "str s10, [x12, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "tbz x14, #0, 100f\n"
+      "str s10, [x13, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 100f\n"
       "96:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 98f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "tbz x13, #1, 97f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "tbz x13, #0, 100f\n"
-      "st1 { v9.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "tbz x14, #2, 98f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "tbz x14, #1, 97f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "tbz x14, #0, 100f\n"
+      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 100f\n"
       "97:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 100f\n"
-      "str s9, [x12, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "tbz x14, #0, 100f\n"
+      "str s9, [x13, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 100f\n"
       "98:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 99f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "tbz x13, #0, 100f\n"
-      "st1 { v8.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "tbz x14, #1, 99f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "tbz x14, #0, 100f\n"
+      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 100f\n"
       "99:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "100:"  // Height 3: Partial direct writeback: Done
       "b 102f\n"
       "101:"  // Height 3: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "102:"  // Height 3: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 70b\n"
       "b 206f\n"
       "103:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "104:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 105f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 105f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 105f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "105:"  // Height 4: B setup done
-      "cbz x14, 106f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 106f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -1355,132 +1355,132 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "b 117f\n"
       "106:"  // Height 4: no bias
       "tbz %x[flags], #0, 116f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 115f\n"
-      "tbz x13, #3, 110f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 108f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 107f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "tbz x13, #0, 114f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "tbz x14, #3, 110f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 108f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 107f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "tbz x14, #0, 114f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 114f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 114f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 114f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 114f\n"
       "108:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x13, #1, 109f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "tbz x13, #0, 114f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "tbz x14, #1, 109f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "tbz x14, #0, 114f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 114f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 114f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 114f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 114f\n"
       "110:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x13, #2, 112f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 111f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "tbz x13, #0, 114f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "tbz x14, #2, 112f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 111f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "tbz x14, #0, 114f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 114f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 114f\n"
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 114f\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 114f\n"
       "112:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x13, #1, 113f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "tbz x13, #0, 114f\n"
-      "ld1 { v8.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "tbz x14, #1, 113f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "tbz x14, #0, 114f\n"
+      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 114f\n"
       "113:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s8, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "114:"  // Height 4: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 117f\n"
       "115:"  // Height 4: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "b 117f\n"
       "116:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1500,217 +1500,217 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "117:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "118:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 119f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 120f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 120f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 120f\n"
       "119:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "120:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 123f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 122f\n"
       "121:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x26, x26, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 121b\n"
       "122:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
@@ -1720,29 +1720,29 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "123:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 125f\n"
+      "cbz x27, 125f\n"
       "124:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x11, x11, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
+      "add x11, x11, #0x10\n"
       "add x10, x10, #0x10\n"
-      "add x9, x9, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
@@ -1751,21 +1751,21 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "cbnz x26, 124b\n"
+      "cbnz x27, 124b\n"
       "125:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 118b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 126f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -1799,160 +1799,160 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v22.4s, v22.4s, v0.4s\n"
       "fmax v23.4s, v23.4s, v0.4s\n"
       "126:"  // Height 4: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 135f\n"
-      "tbz x13, #3, 130f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 128f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 127f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "tbz x13, #0, 134f\n"
-      "st1 { v11.s }[2], [x12]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
+      "tbz x14, #3, 130f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 128f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 127f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "tbz x14, #0, 134f\n"
+      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
       "b 134f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 134f\n"
-      "str s11, [x12, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
+      "tbz x14, #0, 134f\n"
+      "str s11, [x13, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
       "b 134f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 129f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "tbz x13, #0, 134f\n"
-      "st1 { v10.s }[2], [x12]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
+      "tbz x14, #1, 129f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "tbz x14, #0, 134f\n"
+      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
       "b 134f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 134f\n"
-      "str s10, [x12, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
+      "tbz x14, #0, 134f\n"
+      "str s10, [x13, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
       "b 134f\n"
       "130:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 132f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 131f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "tbz x13, #0, 134f\n"
-      "st1 { v9.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
+      "tbz x14, #2, 132f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 131f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "tbz x14, #0, 134f\n"
+      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
       "b 134f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 134f\n"
-      "str s9, [x12, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
+      "tbz x14, #0, 134f\n"
+      "str s9, [x13, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
       "b 134f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 133f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "tbz x13, #0, 134f\n"
-      "st1 { v8.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
+      "tbz x14, #1, 133f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "tbz x14, #0, 134f\n"
+      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
       "b 134f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
       "134:"  // Height 4: Partial direct writeback: Done
       "b 136f\n"
       "135:"  // Height 4: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
       "136:"  // Height 4: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 104b\n"
       "b 206f\n"
       "137:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "138:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 139f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 139f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 139f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "139:"  // Height 5: B setup done
-      "cbz x14, 140f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 140f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -1966,153 +1966,153 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "b 151f\n"
       "140:"  // Height 5: no bias
       "tbz %x[flags], #0, 150f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 149f\n"
-      "tbz x13, #3, 144f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 142f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 141f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "tbz x14, #3, 144f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 142f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 141f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 148f\n"
       "141:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 148f\n"
       "142:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x13, #1, 143f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "tbz x14, #1, 143f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 148f\n"
       "143:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 148f\n"
       "144:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x13, #2, 146f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 145f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "tbz x14, #2, 146f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 145f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 148f\n"
       "145:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 148f\n"
       "146:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x13, #1, 147f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v8.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "tbz x14, #1, 147f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 148f\n"
       "147:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "ldr s8, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "148:"  // Height 5: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 151f\n"
       "149:"  // Height 5: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
       "b 151f\n"
       "150:"  // Height 5: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2136,254 +2136,254 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "151:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "152:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 153f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 154f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 154f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 154f\n"
       "153:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "154:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 157f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 156f\n"
       "155:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "fmla v26.4s, v6.4s, v4.s[3]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 155b\n"
       "156:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
+      "ldr q6, [x12, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "add x11, x11, #0x40\n"
+      "add x12, x12, #0x40\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "add x9, x9, #0x40\n"
+      "add x10, x10, #0x40\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
@@ -2395,32 +2395,32 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "157:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 159f\n"
+      "cbz x27, 159f\n"
       "158:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "add x11, x11, #0x10\n"
+      "add x12, x12, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x10, x10, #0x10\n"
+      "add x11, x11, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "add x9, x9, #0x10\n"
+      "add x10, x10, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
@@ -2431,22 +2431,22 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "cbnz x26, 158b\n"
+      "cbnz x27, 158b\n"
       "159:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 152b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 160f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -2488,183 +2488,183 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v26.4s, v26.4s, v0.4s\n"
       "fmax v27.4s, v27.4s, v0.4s\n"
       "160:"  // Height 5: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 169f\n"
-      "tbz x13, #3, 164f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 162f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 161f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "tbz x13, #0, 168f\n"
-      "st1 { v11.s }[2], [x12]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "tbz x14, #3, 164f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 162f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 161f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "tbz x14, #0, 168f\n"
+      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 168f\n"
       "161:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 168f\n"
-      "str s11, [x12, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "tbz x14, #0, 168f\n"
+      "str s11, [x13, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 168f\n"
       "162:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 163f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "tbz x13, #0, 168f\n"
-      "st1 { v10.s }[2], [x12]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "tbz x14, #1, 163f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "tbz x14, #0, 168f\n"
+      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 168f\n"
       "163:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 168f\n"
-      "str s10, [x12, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "tbz x14, #0, 168f\n"
+      "str s10, [x13, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 168f\n"
       "164:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 166f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 165f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "tbz x13, #0, 168f\n"
-      "st1 { v9.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "tbz x14, #2, 166f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 165f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "tbz x14, #0, 168f\n"
+      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 168f\n"
       "165:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 168f\n"
-      "str s9, [x12, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "tbz x14, #0, 168f\n"
+      "str s9, [x13, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 168f\n"
       "166:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 167f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "tbz x13, #0, 168f\n"
-      "st1 { v8.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "tbz x14, #1, 167f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "tbz x14, #0, 168f\n"
+      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 168f\n"
       "167:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "168:"  // Height 5: Partial direct writeback: Done
       "b 170f\n"
       "169:"  // Height 5: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "170:"  // Height 5: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 138b\n"
       "b 206f\n"
       "171:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x18\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "172:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0xc\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0xc\n"
       "bgt 173f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 173f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 173f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "173:"  // Height 6: B setup done
-      "cbz x14, 174f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 174f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "mov v12.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
-      "add x14, x14, #0x40\n"
+      "add x15, x15, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v20.16b, v8.16b\n"
@@ -2682,174 +2682,174 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "b 185f\n"
       "174:"  // Height 6: no bias
       "tbz %x[flags], #0, 184f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "cmp x13, #0x10\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x14, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 183f\n"
-      "tbz x13, #3, 178f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 176f\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 175f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
-      "tbz x13, #0, 182f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "tbz x14, #3, 178f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 176f\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 175f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
+      "tbz x14, #0, 182f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 182f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 182f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 182f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 182f\n"
       "176:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x13, #1, 177f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
-      "tbz x13, #0, 182f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "tbz x14, #1, 177f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
+      "tbz x14, #0, 182f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 182f\n"
       "177:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 182f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 182f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 182f\n"
       "178:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x13, #2, 180f\n"
-      "ld1 { v8.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 179f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
-      "tbz x13, #0, 182f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "tbz x14, #2, 180f\n"
+      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 179f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
+      "tbz x14, #0, 182f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 182f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 182f\n"
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 182f\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 182f\n"
       "180:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x13, #1, 181f\n"
-      "ldr d8, [x12], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
-      "tbz x13, #0, 182f\n"
-      "ld1 { v8.s }[2], [x12]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "tbz x14, #1, 181f\n"
+      "ldr d8, [x13], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
+      "tbz x14, #0, 182f\n"
+      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 182f\n"
       "181:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x12, #0x0]\n"
-      "ldr s12, [x24, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s8, [x13, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "182:"  // Height 6: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 185f\n"
       "183:"  // Height 6: full accumulate
-      "ldr q8, [x12, #0x0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q8, [x13, #0x0]\n"
+      "ldr q9, [x13, #0x10]\n"
+      "ldr q10, [x13, #0x20]\n"
+      "ldr q11, [x13, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "b 185f\n"
       "184:"  // Height 6: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2877,291 +2877,291 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "185:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "186:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 187f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 188f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 188f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 188f\n"
       "187:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "188:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 191f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "blt 190f\n"
       "189:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
       "fmla v28.4s, v6.4s, v5.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
       "fmla v29.4s, v7.4s, v5.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
       "fmla v30.4s, v6.4s, v5.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
       "fmla v31.4s, v7.4s, v5.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
       "fmla v28.4s, v6.4s, v5.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
       "fmla v29.4s, v7.4s, v5.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
       "fmla v30.4s, v6.4s, v5.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
       "fmla v31.4s, v7.4s, v5.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
       "fmla v28.4s, v6.4s, v5.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
       "fmla v29.4s, v7.4s, v5.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "fmla v26.4s, v6.4s, v4.s[3]\n"
       "fmla v30.4s, v6.4s, v5.s[3]\n"
-      "ldr q6, [x11, #0x0]\n"
+      "ldr q6, [x12, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "bge 189b\n"
       "190:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x26, x26, #0x4\n"
-      "add x25, x25, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
+      "add x25, x25, #0x10\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
+      "add x22, x22, #0x10\n"
       "add x21, x21, #0x10\n"
-      "add x20, x20, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
-      "ldr q6, [x11, #0x10]\n"
+      "ldr q6, [x12, #0x10]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q7, [x11, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
       "fmla v28.4s, v6.4s, v5.s[1]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
       "fmla v29.4s, v7.4s, v5.s[1]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
       "fmla v30.4s, v6.4s, v5.s[1]\n"
-      "ldr q6, [x11, #0x20]\n"
+      "ldr q6, [x12, #0x20]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
       "fmla v31.4s, v7.4s, v5.s[1]\n"
-      "ldr q7, [x10, #0x20]\n"
+      "ldr q7, [x11, #0x20]\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
       "fmla v28.4s, v6.4s, v5.s[2]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
       "fmla v29.4s, v7.4s, v5.s[2]\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
       "fmla v30.4s, v6.4s, v5.s[2]\n"
-      "ldr q6, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q6, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
       "fmla v31.4s, v7.4s, v5.s[2]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q7, [x11, #0x30]\n"
+      "add x11, x11, #0x40\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
       "fmla v28.4s, v6.4s, v5.s[3]\n"
-      "ldr q6, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
       "fmla v29.4s, v7.4s, v5.s[3]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
@@ -3175,35 +3175,35 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
       "191:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 193f\n"
+      "cbz x27, 193f\n"
       "192:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
-      "ldr q6, [x11, #0x0]\n"
-      "ldr q7, [x10, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
+      "ldr q6, [x12, #0x0]\n"
+      "ldr q7, [x11, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
+      "add x12, x12, #0x10\n"
       "add x11, x11, #0x10\n"
-      "add x10, x10, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "add x10, x10, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "ldr q7, [x9, #0x0]\n"
+      "add x9, x9, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
@@ -3216,23 +3216,23 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
-      "cbnz x26, 192b\n"
+      "cbnz x27, 192b\n"
       "193:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 186b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 194f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -3282,178 +3282,178 @@ void a64_ffhybrid_fp32_mla_6x16 (
       "fmax v30.4s, v30.4s, v0.4s\n"
       "fmax v31.4s, v31.4s, v0.4s\n"
       "194:"  // Height 6: No activation
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "bge 203f\n"
-      "tbz x13, #3, 198f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 196f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v30.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 195f\n"
-      "str d11, [x12], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "st1 { v11.s }[2], [x12]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
-      "st1 { v31.s }[2], [x20]\n"
+      "tbz x14, #3, 198f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 196f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v30.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 195f\n"
+      "str d11, [x13], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
+      "st1 { v31.s }[2], [x21]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 202f\n"
-      "str s11, [x12, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
-      "str s31, [x20, #0x0]\n"
+      "tbz x14, #0, 202f\n"
+      "str s11, [x13, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
+      "str s31, [x21, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 197f\n"
-      "str d10, [x12], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "st1 { v10.s }[2], [x12]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
+      "tbz x14, #1, 197f\n"
+      "str d10, [x13], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 202f\n"
-      "str s10, [x12, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
+      "tbz x14, #0, 202f\n"
+      "str s10, [x13, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 200f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 199f\n"
-      "str d9, [x12], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "st1 { v9.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
+      "tbz x14, #2, 200f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 199f\n"
+      "str d9, [x13], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 202f\n"
-      "str s9, [x12, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
+      "tbz x14, #0, 202f\n"
+      "str s9, [x13, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 201f\n"
-      "str d8, [x12], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
-      "tbz x13, #0, 202f\n"
-      "st1 { v8.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
+      "tbz x14, #1, 201f\n"
+      "str d8, [x13], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
+      "tbz x14, #0, 202f\n"
+      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
       "202:"  // Height 6: Partial direct writeback: Done
       "b 204f\n"
       "203:"  // Height 6: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
-      "str q28, [x20, #0x0]\n"
-      "str q29, [x20, #0x10]\n"
-      "str q30, [x20, #0x20]\n"
-      "str q31, [x20, #0x30]\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "add x13, x13, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
+      "str q28, [x21, #0x0]\n"
+      "str q29, [x21, #0x10]\n"
+      "str q30, [x21, #0x20]\n"
+      "str q31, [x21, #0x30]\n"
       "204:"  // Height 6: Writeback done
-      "subs x13, x13, #0x10\n"
+      "subs x14, x14, #0x10\n"
       "bgt 172b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 206f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 205f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "205:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "206:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp
index 245e653a43..1f707fa962 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -102,53 +102,53 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "cmp %x[M], #0x2\n"
       "bgt 89f\n"
       "beq 45f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x14\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x14\n"
       "bgt 3f\n"
-      "cmp x13, #0x10\n"
-      "mov x26, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x27, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0xc\n"
-      "mov x27, x11\n"
+      "cmp x14, #0xc\n"
+      "mov x28, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "cbz x14, 4f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 4f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x14, #0x40]\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q12, [x15, #0x40]\n"
+      "ldr q13, [x15, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
-      "add x14, x14, #0x60\n"
+      "add x15, x15, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -156,101 +156,101 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "b 20f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 19f\n"
-      "cmp x13, #0x18\n"
+      "cmp x14, #0x18\n"
       "bge 17f\n"
-      "tbz x13, #4, 8f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v12.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 6f\n"
-      "ld1 { v13.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 5f\n"
-      "ldr d20, [x12], #0x8\n"
-      "mov x19, #0x58\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v20.s }[2], [x12]\n"
+      "tbz x14, #4, 8f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 6f\n"
+      "ld1 { v13.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 5f\n"
+      "ldr d20, [x13], #0x8\n"
+      "mov x20, #0x58\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v20.s }[2], [x13]\n"
       "b 16f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
-      "tbz x13, #0, 16f\n"
-      "ldr s20, [x12, #0x0]\n"
+      "mov x20, #0x50\n"
+      "tbz x14, #0, 16f\n"
+      "ldr s20, [x13, #0x0]\n"
       "b 16f\n"
       "6:"  // Height 1: Partial accumulate: partial_2_16
-      "tbz x13, #1, 7f\n"
-      "ldr d13, [x12], #0x8\n"
-      "mov x19, #0x48\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v13.s }[2], [x12]\n"
+      "tbz x14, #1, 7f\n"
+      "ldr d13, [x13], #0x8\n"
+      "mov x20, #0x48\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v13.s }[2], [x13]\n"
       "b 16f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
-      "tbz x13, #0, 16f\n"
-      "ldr s13, [x12, #0x0]\n"
+      "mov x20, #0x40\n"
+      "tbz x14, #0, 16f\n"
+      "ldr s13, [x13, #0x0]\n"
       "b 16f\n"
       "8:"  // Height 1: Partial accumulate: partial_8_0
-      "tbz x13, #3, 12f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 10f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 9f\n"
-      "ldr d12, [x12], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v12.s }[2], [x12]\n"
+      "tbz x14, #3, 12f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 10f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 9f\n"
+      "ldr d12, [x13], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v12.s }[2], [x13]\n"
       "b 16f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 16f\n"
-      "ldr s12, [x12, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 16f\n"
+      "ldr s12, [x13, #0x0]\n"
       "b 16f\n"
       "10:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x13, #1, 11f\n"
-      "ldr d11, [x12], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v11.s }[2], [x12]\n"
+      "tbz x14, #1, 11f\n"
+      "ldr d11, [x13], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v11.s }[2], [x13]\n"
       "b 16f\n"
       "11:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 16f\n"
-      "ldr s11, [x12, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 16f\n"
+      "ldr s11, [x13, #0x0]\n"
       "b 16f\n"
       "12:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x13, #2, 14f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 13f\n"
-      "ldr d10, [x12], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v10.s }[2], [x12]\n"
+      "tbz x14, #2, 14f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 13f\n"
+      "ldr d10, [x13], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v10.s }[2], [x13]\n"
       "b 16f\n"
       "13:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 16f\n"
-      "ldr s10, [x12, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 16f\n"
+      "ldr s10, [x13, #0x0]\n"
       "b 16f\n"
       "14:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x13, #1, 15f\n"
-      "ldr d9, [x12], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 16f\n"
-      "ld1 { v9.s }[2], [x12]\n"
+      "tbz x14, #1, 15f\n"
+      "ldr d9, [x13], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 16f\n"
+      "ld1 { v9.s }[2], [x13]\n"
       "b 16f\n"
       "15:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x13, #0x0]\n"
+      "mov x20, #0x0\n"
       "16:"  // Height 1: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 18f\n"
       "17:"  // Height 1: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q12, [x12, #0x30]\n"
-      "ldr q13, [x12, #0x40]\n"
-      "ldr q20, [x12, #0x50]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q12, [x13, #0x30]\n"
+      "ldr q13, [x13, #0x40]\n"
+      "ldr q20, [x13, #0x50]\n"
       "18:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -279,142 +279,142 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "20:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "21:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 22f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 23f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 23f\n"
       "22:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "23:"  // Height 1: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 26f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "ldr q4, [x11, #0x0]\n"
-      "cmp x24, #0x8\n"
-      "ldr q5, [x11, #0x10]\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ldr q4, [x12, #0x0]\n"
+      "cmp x25, #0x8\n"
+      "ldr q5, [x12, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       "blt 25f\n"
       "24:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x8\n"
+      "ldr q7, [x27, #0x10]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
+      "add x12, x12, #0x20\n"
+      "ldr q4, [x12, #0x0]\n"
       "add x11, x11, #0x20\n"
-      "ldr q4, [x11, #0x0]\n"
-      "add x10, x10, #0x20\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ldr q7, [x11, #0x10]\n"
+      "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "bge 24b\n"
       "25:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "sub x24, x24, #0x4\n"
+      "ldr q7, [x27, #0x10]\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
-      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "26:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 29f\n"
-      "cbz x24, 29f\n"
-      "tbz x24, #1, 27f\n"
-      "ldr d0, [x23], #0x8\n"
-      "tbz x24, #0, 28f\n"
-      "ld1 { v0.s }[2], [x23]\n"
+      "cbz x25, 29f\n"
+      "cbz x25, 29f\n"
+      "tbz x25, #1, 27f\n"
+      "ldr d0, [x24], #0x8\n"
+      "tbz x25, #0, 28f\n"
+      "ld1 { v0.s }[2], [x24]\n"
       "b 28f\n"
       "27:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
       "28:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q4, [x9, #0x0]\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q4, [x10, #0x0]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q4, [x27, #0x0]\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x26, #0x0]\n"
-      "ldr q7, [x26, #0x10]\n"
+      "ldr q6, [x27, #0x0]\n"
+      "ldr q7, [x27, #0x10]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "29:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 21b\n"
       "uzp1 v8.2d, v8.2d, v14.2d\n"
       "uzp1 v9.2d, v9.2d, v15.2d\n"
@@ -423,10 +423,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v12.2d, v12.2d, v18.2d\n"
       "uzp1 v13.2d, v13.2d, v19.2d\n"
       "tbz %x[flags], #1, 30f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v1.4s\n"
       "fmin v9.4s, v9.4s, v1.4s\n"
       "fmin v10.4s, v10.4s, v1.4s\n"
@@ -440,141 +440,141 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "fmax v12.4s, v12.4s, v0.4s\n"
       "fmax v13.4s, v13.4s, v0.4s\n"
       "30:"  // Height 1: No activation
-      "cmp x13, #0x18\n"
+      "cmp x14, #0x18\n"
       "bge 43f\n"
-      "tbz x13, #4, 34f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "st1 { v11.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 32f\n"
-      "st1 { v12.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 31f\n"
-      "str d13, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v13.s }[2], [x12]\n"
+      "tbz x14, #4, 34f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v11.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 32f\n"
+      "st1 { v12.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 31f\n"
+      "str d13, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v13.s }[2], [x13]\n"
       "b 42f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 42f\n"
-      "str s13, [x12, #0x0]\n"
+      "tbz x14, #0, 42f\n"
+      "str s13, [x13, #0x0]\n"
       "b 42f\n"
       "32:"  // Height 1: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 33f\n"
-      "str d12, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v12.s }[2], [x12]\n"
+      "tbz x14, #1, 33f\n"
+      "str d12, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v12.s }[2], [x13]\n"
       "b 42f\n"
       "33:"  // Height 1: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 42f\n"
-      "str s12, [x12, #0x0]\n"
+      "tbz x14, #0, 42f\n"
+      "str s12, [x13, #0x0]\n"
       "b 42f\n"
       "34:"  // Height 1: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 38f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "st1 { v9.4s }, [x12], #0x10\n"
-      "tbz x13, #2, 36f\n"
-      "st1 { v10.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 35f\n"
-      "str d11, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v11.s }[2], [x12]\n"
+      "tbz x14, #3, 38f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v9.4s }, [x13], #0x10\n"
+      "tbz x14, #2, 36f\n"
+      "st1 { v10.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 35f\n"
+      "str d11, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v11.s }[2], [x13]\n"
       "b 42f\n"
       "35:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 42f\n"
-      "str s11, [x12, #0x0]\n"
+      "tbz x14, #0, 42f\n"
+      "str s11, [x13, #0x0]\n"
       "b 42f\n"
       "36:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 37f\n"
-      "str d10, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v10.s }[2], [x12]\n"
+      "tbz x14, #1, 37f\n"
+      "str d10, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v10.s }[2], [x13]\n"
       "b 42f\n"
       "37:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 42f\n"
-      "str s10, [x12, #0x0]\n"
+      "tbz x14, #0, 42f\n"
+      "str s10, [x13, #0x0]\n"
       "b 42f\n"
       "38:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 40f\n"
-      "st1 { v8.4s }, [x12], #0x10\n"
-      "tbz x13, #1, 39f\n"
-      "str d9, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v9.s }[2], [x12]\n"
+      "tbz x14, #2, 40f\n"
+      "st1 { v8.4s }, [x13], #0x10\n"
+      "tbz x14, #1, 39f\n"
+      "str d9, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v9.s }[2], [x13]\n"
       "b 42f\n"
       "39:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 42f\n"
-      "str s9, [x12, #0x0]\n"
+      "tbz x14, #0, 42f\n"
+      "str s9, [x13, #0x0]\n"
       "b 42f\n"
       "40:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 41f\n"
-      "str d8, [x12], #0x8\n"
-      "tbz x13, #0, 42f\n"
-      "st1 { v8.s }[2], [x12]\n"
+      "tbz x14, #1, 41f\n"
+      "str d8, [x13], #0x8\n"
+      "tbz x14, #0, 42f\n"
+      "st1 { v8.s }[2], [x13]\n"
       "b 42f\n"
       "41:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x12, #0x0]\n"
+      "str s8, [x13, #0x0]\n"
       "42:"  // Height 1: Partial direct writeback: Done
       "b 44f\n"
       "43:"  // Height 1: Full writeback
-      "str q8, [x12, #0x0]\n"
-      "str q9, [x12, #0x10]\n"
-      "str q10, [x12, #0x20]\n"
-      "str q11, [x12, #0x30]\n"
-      "str q12, [x12, #0x40]\n"
-      "str q13, [x12, #0x50]\n"
-      "add x12, x12, #0x60\n"
+      "str q8, [x13, #0x0]\n"
+      "str q9, [x13, #0x10]\n"
+      "str q10, [x13, #0x20]\n"
+      "str q11, [x13, #0x30]\n"
+      "str q12, [x13, #0x40]\n"
+      "str q13, [x13, #0x50]\n"
+      "add x13, x13, #0x60\n"
       "44:"  // Height 1: Writeback done
-      "subs x13, x13, #0x18\n"
+      "subs x14, x14, #0x18\n"
       "bgt 2b\n"
       "b 178f\n"
       "45:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "46:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x14\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x14\n"
       "bgt 47f\n"
-      "cmp x13, #0x10\n"
-      "mov x26, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x27, x12\n"
       "bgt 47f\n"
-      "cmp x13, #0xc\n"
-      "mov x27, x11\n"
+      "cmp x14, #0xc\n"
+      "mov x28, x12\n"
       "bgt 47f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 47f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 47f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "47:"  // Height 2: B setup done
-      "cbz x14, 48f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 48f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x14, #0x40]\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q12, [x15, #0x40]\n"
+      "ldr q13, [x15, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
-      "add x14, x14, #0x60\n"
+      "add x15, x15, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -582,136 +582,136 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "b 64f\n"
       "48:"  // Height 2: no bias
       "tbz %x[flags], #0, 63f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x13, #0x18\n"
-      "add x22, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x14, #0x18\n"
+      "add x23, x13, x20, LSL #2\n"
       "bge 61f\n"
-      "tbz x13, #4, 52f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v12.4s }, [x12], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 50f\n"
-      "ld1 { v13.4s }, [x12], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 49f\n"
-      "ldr d20, [x12], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "mov x19, #0x58\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v20.s }[2], [x12]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x14, #4, 52f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 50f\n"
+      "ld1 { v13.4s }, [x13], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 49f\n"
+      "ldr d20, [x13], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v20.s }[2], [x13]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 60f\n"
       "49:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
-      "tbz x13, #0, 60f\n"
-      "ldr s20, [x12, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x20, #0x50\n"
+      "tbz x14, #0, 60f\n"
+      "ldr s20, [x13, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 60f\n"
       "50:"  // Height 2: Partial accumulate: partial_2_16
-      "tbz x13, #1, 51f\n"
-      "ldr d13, [x12], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v13.s }[2], [x12]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x14, #1, 51f\n"
+      "ldr d13, [x13], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 60f\n"
       "51:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
-      "tbz x13, #0, 60f\n"
-      "ldr s13, [x12, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x20, #0x40\n"
+      "tbz x14, #0, 60f\n"
+      "ldr s13, [x13, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 60f\n"
       "52:"  // Height 2: Partial accumulate: partial_8_0
-      "tbz x13, #3, 56f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 54f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 53f\n"
-      "ldr d12, [x12], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "mov x19, #0x38\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v12.s }[2], [x12]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x14, #3, 56f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 54f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 53f\n"
+      "ldr d12, [x13], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 60f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 60f\n"
-      "ldr s12, [x12, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 60f\n"
+      "ldr s12, [x13, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 60f\n"
       "54:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x13, #1, 55f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "tbz x14, #1, 55f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 60f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 60f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 60f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
       "b 60f\n"
       "56:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x13, #2, 58f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 57f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "mov x19, #0x18\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x22]\n"
+      "tbz x14, #2, 58f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 57f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x23]\n"
       "b 60f\n"
       "57:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 60f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 60f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
       "b 60f\n"
       "58:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x13, #1, 59f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "tbz x13, #0, 60f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x22]\n"
+      "tbz x14, #1, 59f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "tbz x14, #0, 60f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x23]\n"
       "b 60f\n"
       "59:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s14, [x22, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
       "60:"  // Height 2: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 62f\n"
       "61:"  // Height 2: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q12, [x12, #0x30]\n"
-      "ldr q13, [x12, #0x40]\n"
-      "ldr q20, [x12, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q12, [x13, #0x30]\n"
+      "ldr q13, [x13, #0x40]\n"
+      "ldr q20, [x13, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
       "62:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -740,158 +740,158 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "64:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "65:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 66f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 67f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 67f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 67f\n"
       "66:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
       "67:"  // Height 2: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 70f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "cmp x24, #0x8\n"
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       "blt 69f\n"
       "68:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x8\n"
-      "add x11, x11, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x11, #0x0]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q4, [x12, #0x0]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "add x10, x10, #0x20\n"
+      "ldr q7, [x11, #0x10]\n"
       "add x9, x9, #0x20\n"
-      "ldr q7, [x10, #0x10]\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "bge 68b\n"
       "69:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "sub x24, x24, #0x4\n"
+      "ldr q7, [x27, #0x10]\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
-      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "70:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 73f\n"
-      "cbz x24, 73f\n"
-      "tbz x24, #1, 71f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "tbz x24, #0, 72f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
+      "cbz x25, 73f\n"
+      "cbz x25, 73f\n"
+      "tbz x25, #1, 71f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "tbz x25, #0, 72f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
       "b 72f\n"
       "71:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
       "72:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q4, [x9, #0x0]\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q4, [x10, #0x0]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q4, [x27, #0x0]\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q6, [x26, #0x0]\n"
-      "ldr q7, [x26, #0x10]\n"
+      "ldr q6, [x27, #0x0]\n"
+      "ldr q7, [x27, #0x10]\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x12, x12, #0x20\n"
       "add x11, x11, #0x20\n"
       "add x10, x10, #0x20\n"
       "add x9, x9, #0x20\n"
       "add x28, x28, #0x20\n"
       "add x27, x27, #0x20\n"
-      "add x26, x26, #0x20\n"
       "73:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 65b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
-      "add x22, x12, x19, LSL #2\n"
+      "add x23, x13, x20, LSL #2\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
@@ -903,10 +903,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v18.2d, v13.2d, v19.2d\n"
       "uzp2 v13.2d, v13.2d, v19.2d\n"
       "tbz %x[flags], #1, 74f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v4.4s, v4.4s, v1.4s\n"
       "fmin v14.4s, v14.4s, v1.4s\n"
       "fmin v15.4s, v15.4s, v1.4s\n"
@@ -932,174 +932,174 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "fmax v12.4s, v12.4s, v0.4s\n"
       "fmax v13.4s, v13.4s, v0.4s\n"
       "74:"  // Height 2: No activation
-      "cmp x13, #0x18\n"
+      "cmp x14, #0x18\n"
       "bge 87f\n"
-      "tbz x13, #4, 78f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v16.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 76f\n"
-      "st1 { v17.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 75f\n"
-      "str d18, [x12], #0x8\n"
-      "str d13, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v18.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x22]\n"
+      "tbz x14, #4, 78f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v16.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 76f\n"
+      "st1 { v17.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 75f\n"
+      "str d18, [x13], #0x8\n"
+      "str d13, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v18.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x23]\n"
       "b 86f\n"
       "75:"  // Height 2: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 86f\n"
-      "str s18, [x12, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
+      "tbz x14, #0, 86f\n"
+      "str s18, [x13, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
       "b 86f\n"
       "76:"  // Height 2: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 77f\n"
-      "str d17, [x12], #0x8\n"
-      "str d12, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v17.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x22]\n"
+      "tbz x14, #1, 77f\n"
+      "str d17, [x13], #0x8\n"
+      "str d12, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v17.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x23]\n"
       "b 86f\n"
       "77:"  // Height 2: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 86f\n"
-      "str s17, [x12, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
+      "tbz x14, #0, 86f\n"
+      "str s17, [x13, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
       "b 86f\n"
       "78:"  // Height 2: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 82f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "tbz x13, #2, 80f\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 79f\n"
-      "str d16, [x12], #0x8\n"
-      "str d11, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v16.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x22]\n"
+      "tbz x14, #3, 82f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "tbz x14, #2, 80f\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 79f\n"
+      "str d16, [x13], #0x8\n"
+      "str d11, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v16.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x23]\n"
       "b 86f\n"
       "79:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 86f\n"
-      "str s16, [x12, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
+      "tbz x14, #0, 86f\n"
+      "str s16, [x13, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
       "b 86f\n"
       "80:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 81f\n"
-      "str d15, [x12], #0x8\n"
-      "str d10, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v15.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x22]\n"
+      "tbz x14, #1, 81f\n"
+      "str d15, [x13], #0x8\n"
+      "str d10, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v15.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x23]\n"
       "b 86f\n"
       "81:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 86f\n"
-      "str s15, [x12, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
+      "tbz x14, #0, 86f\n"
+      "str s15, [x13, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
       "b 86f\n"
       "82:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 84f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "tbz x13, #1, 83f\n"
-      "str d14, [x12], #0x8\n"
-      "str d9, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x22]\n"
+      "tbz x14, #2, 84f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "tbz x14, #1, 83f\n"
+      "str d14, [x13], #0x8\n"
+      "str d9, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x23]\n"
       "b 86f\n"
       "83:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 86f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
+      "tbz x14, #0, 86f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
       "b 86f\n"
       "84:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 85f\n"
-      "str d4, [x12], #0x8\n"
-      "str d8, [x22], #0x8\n"
-      "tbz x13, #0, 86f\n"
-      "st1 { v4.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x22]\n"
+      "tbz x14, #1, 85f\n"
+      "str d4, [x13], #0x8\n"
+      "str d8, [x23], #0x8\n"
+      "tbz x14, #0, 86f\n"
+      "st1 { v4.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x23]\n"
       "b 86f\n"
       "85:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s4, [x12, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
+      "str s4, [x13, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
       "86:"  // Height 2: Partial direct writeback: Done
       "b 88f\n"
       "87:"  // Height 2: Full writeback
-      "str q4, [x12, #0x0]\n"
-      "str q14, [x12, #0x10]\n"
-      "str q15, [x12, #0x20]\n"
-      "str q16, [x12, #0x30]\n"
-      "str q17, [x12, #0x40]\n"
-      "str q18, [x12, #0x50]\n"
-      "add x12, x12, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
+      "str q4, [x13, #0x0]\n"
+      "str q14, [x13, #0x10]\n"
+      "str q15, [x13, #0x20]\n"
+      "str q16, [x13, #0x30]\n"
+      "str q17, [x13, #0x40]\n"
+      "str q18, [x13, #0x50]\n"
+      "add x13, x13, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
       "88:"  // Height 2: Writeback done
-      "subs x13, x13, #0x18\n"
+      "subs x14, x14, #0x18\n"
       "bgt 46b\n"
       "b 178f\n"
       "89:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "90:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x14\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x14\n"
       "bgt 91f\n"
-      "cmp x13, #0x10\n"
-      "mov x26, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x27, x12\n"
       "bgt 91f\n"
-      "cmp x13, #0xc\n"
-      "mov x27, x11\n"
+      "cmp x14, #0xc\n"
+      "mov x28, x12\n"
       "bgt 91f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 91f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 91f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "91:"  // Height 3: B setup done
-      "cbz x14, 92f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 92f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x14, #0x40]\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q12, [x15, #0x40]\n"
+      "ldr q13, [x15, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
-      "add x14, x14, #0x60\n"
+      "add x15, x15, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -1119,170 +1119,170 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "b 108f\n"
       "92:"  // Height 3: no bias
       "tbz %x[flags], #0, 107f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "cmp x13, #0x18\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "cmp x14, #0x18\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 105f\n"
-      "tbz x13, #4, 96f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v12.4s }, [x12], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 94f\n"
-      "ld1 { v13.4s }, [x12], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 93f\n"
-      "ldr d20, [x12], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "mov x19, #0x58\n"
-      "ldr d4, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v20.s }[2], [x12]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
+      "tbz x14, #4, 96f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 94f\n"
+      "ld1 { v13.4s }, [x13], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 93f\n"
+      "ldr d20, [x13], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d4, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v20.s }[2], [x13]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
       "b 104f\n"
       "93:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
-      "tbz x13, #0, 104f\n"
-      "ldr s20, [x12, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
+      "mov x20, #0x50\n"
+      "tbz x14, #0, 104f\n"
+      "ldr s20, [x13, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
       "b 104f\n"
       "94:"  // Height 3: Partial accumulate: partial_2_16
-      "tbz x13, #1, 95f\n"
-      "ldr d13, [x12], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d25, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v13.s }[2], [x12]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "tbz x14, #1, 95f\n"
+      "ldr d13, [x13], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d25, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 104f\n"
       "95:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
-      "tbz x13, #0, 104f\n"
-      "ldr s13, [x12, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "mov x20, #0x40\n"
+      "tbz x14, #0, 104f\n"
+      "ldr s13, [x13, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 104f\n"
       "96:"  // Height 3: Partial accumulate: partial_8_0
-      "tbz x13, #3, 100f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 98f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 97f\n"
-      "ldr d12, [x12], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v12.s }[2], [x12]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "tbz x14, #3, 100f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 98f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 97f\n"
+      "ldr d12, [x13], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 104f\n"
       "97:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 104f\n"
-      "ldr s12, [x12, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 104f\n"
+      "ldr s12, [x13, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "b 104f\n"
       "98:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x13, #1, 99f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x14, #1, 99f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 104f\n"
       "99:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 104f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 104f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 104f\n"
       "100:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x13, #2, 102f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 101f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x14, #2, 102f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 101f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 104f\n"
       "101:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 104f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 104f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 104f\n"
       "102:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x13, #1, 103f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x13, #0, 104f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x14, #1, 103f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x14, #0, 104f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 104f\n"
       "103:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s14, [x22, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s21, [x21, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s21, [x22, #0x0]\n"
       "104:"  // Height 3: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 106f\n"
       "105:"  // Height 3: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q12, [x12, #0x30]\n"
-      "ldr q13, [x12, #0x40]\n"
-      "ldr q20, [x12, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q21, [x21, #0x0]\n"
-      "ldr q22, [x21, #0x10]\n"
-      "ldr q23, [x21, #0x20]\n"
-      "ldr q24, [x21, #0x30]\n"
-      "ldr q25, [x21, #0x40]\n"
-      "ldr q4, [x21, #0x50]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q12, [x13, #0x30]\n"
+      "ldr q13, [x13, #0x40]\n"
+      "ldr q20, [x13, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q21, [x22, #0x0]\n"
+      "ldr q22, [x22, #0x10]\n"
+      "ldr q23, [x22, #0x20]\n"
+      "ldr q24, [x22, #0x30]\n"
+      "ldr q25, [x22, #0x40]\n"
+      "ldr q4, [x22, #0x50]\n"
       "106:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -1335,126 +1335,126 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "108:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "109:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 110f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 111f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 111f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 111f\n"
       "110:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "111:"  // Height 3: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 114f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "cmp x24, #0x8\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       "blt 113f\n"
       "112:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "sub x24, x24, #0x4\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "cmp x24, #0x8\n"
+      "cmp x25, #0x8\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "add x27, x27, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "add x26, x26, #0x20\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x11, #0x0]\n"
+      "ldr q4, [x12, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec59  // bfmmla v25.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q7, [x11, #0x10]\n"
       "bge 112b\n"
       "113:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "sub x24, x24, #0x4\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x27, x27, #0x20\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "add x26, x26, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
@@ -1464,60 +1464,60 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "114:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 117f\n"
-      "cbz x24, 117f\n"
-      "tbz x24, #1, 115f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "ldr d2, [x21], #0x8\n"
-      "tbz x24, #0, 116f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
-      "ld1 { v2.s }[2], [x21]\n"
+      "cbz x25, 117f\n"
+      "cbz x25, 117f\n"
+      "tbz x25, #1, 115f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "ldr d2, [x22], #0x8\n"
+      "tbz x25, #0, 116f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
+      "ld1 { v2.s }[2], [x22]\n"
       "b 116f\n"
       "115:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
-      "ldr s2, [x21, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
+      "ldr s2, [x22, #0x0]\n"
       "116:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "add x11, x11, #0x20\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x27, x27, #0x20\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
+      "ldr q7, [x27, #0x10]\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "add x26, x26, #0x20\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
@@ -1526,17 +1526,17 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "117:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 109b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
       "uzp2 v10.2d, v10.2d, v16.2d\n"
       "uzp1 v16.2d, v11.2d, v17.2d\n"
@@ -1552,10 +1552,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v24.2d, v24.2d, v30.2d\n"
       "uzp1 v25.2d, v25.2d, v31.2d\n"
       "tbz %x[flags], #1, 118f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v4.4s, v4.4s, v1.4s\n"
       "fmin v14.4s, v14.4s, v1.4s\n"
       "fmin v15.4s, v15.4s, v1.4s\n"
@@ -1593,210 +1593,210 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "fmax v24.4s, v24.4s, v0.4s\n"
       "fmax v25.4s, v25.4s, v0.4s\n"
       "118:"  // Height 3: No activation
-      "cmp x13, #0x18\n"
+      "cmp x14, #0x18\n"
       "bge 131f\n"
-      "tbz x13, #4, 122f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v16.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 120f\n"
-      "st1 { v17.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 119f\n"
-      "str d18, [x12], #0x8\n"
-      "str d13, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v18.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "tbz x14, #4, 122f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v16.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 120f\n"
+      "st1 { v17.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 119f\n"
+      "str d18, [x13], #0x8\n"
+      "str d13, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v18.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 130f\n"
       "119:"  // Height 3: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 130f\n"
-      "str s18, [x12, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "tbz x14, #0, 130f\n"
+      "str s18, [x13, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 130f\n"
       "120:"  // Height 3: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 121f\n"
-      "str d17, [x12], #0x8\n"
-      "str d12, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v17.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "tbz x14, #1, 121f\n"
+      "str d17, [x13], #0x8\n"
+      "str d12, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v17.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 130f\n"
       "121:"  // Height 3: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 130f\n"
-      "str s17, [x12, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "tbz x14, #0, 130f\n"
+      "str s17, [x13, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "b 130f\n"
       "122:"  // Height 3: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 126f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "tbz x13, #2, 124f\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 123f\n"
-      "str d16, [x12], #0x8\n"
-      "str d11, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v16.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "tbz x14, #3, 126f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "tbz x14, #2, 124f\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 123f\n"
+      "str d16, [x13], #0x8\n"
+      "str d11, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v16.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 130f\n"
       "123:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 130f\n"
-      "str s16, [x12, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "tbz x14, #0, 130f\n"
+      "str s16, [x13, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 130f\n"
       "124:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 125f\n"
-      "str d15, [x12], #0x8\n"
-      "str d10, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v15.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "tbz x14, #1, 125f\n"
+      "str d15, [x13], #0x8\n"
+      "str d10, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v15.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 130f\n"
       "125:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 130f\n"
-      "str s15, [x12, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "tbz x14, #0, 130f\n"
+      "str s15, [x13, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 130f\n"
       "126:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 128f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "tbz x13, #1, 127f\n"
-      "str d14, [x12], #0x8\n"
-      "str d9, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "tbz x14, #2, 128f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "tbz x14, #1, 127f\n"
+      "str d14, [x13], #0x8\n"
+      "str d9, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 130f\n"
       "127:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 130f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "tbz x14, #0, 130f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 130f\n"
       "128:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 129f\n"
-      "str d4, [x12], #0x8\n"
-      "str d8, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x13, #0, 130f\n"
-      "st1 { v4.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "tbz x14, #1, 129f\n"
+      "str d4, [x13], #0x8\n"
+      "str d8, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x14, #0, 130f\n"
+      "st1 { v4.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 130f\n"
       "129:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s4, [x12, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s4, [x13, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "130:"  // Height 3: Partial direct writeback: Done
       "b 132f\n"
       "131:"  // Height 3: Full writeback
-      "str q4, [x12, #0x0]\n"
-      "str q14, [x12, #0x10]\n"
-      "str q15, [x12, #0x20]\n"
-      "str q16, [x12, #0x30]\n"
-      "str q17, [x12, #0x40]\n"
-      "str q18, [x12, #0x50]\n"
-      "add x12, x12, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x21, #0x40]\n"
-      "str q25, [x21, #0x50]\n"
+      "str q4, [x13, #0x0]\n"
+      "str q14, [x13, #0x10]\n"
+      "str q15, [x13, #0x20]\n"
+      "str q16, [x13, #0x30]\n"
+      "str q17, [x13, #0x40]\n"
+      "str q18, [x13, #0x50]\n"
+      "add x13, x13, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x22, #0x40]\n"
+      "str q25, [x22, #0x50]\n"
       "132:"  // Height 3: Writeback done
-      "subs x13, x13, #0x18\n"
+      "subs x14, x14, #0x18\n"
       "bgt 90b\n"
       "b 178f\n"
       "133:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x10\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x10\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "134:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x13, #0x14\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x14, #0x14\n"
       "bgt 135f\n"
-      "cmp x13, #0x10\n"
-      "mov x26, x11\n"
+      "cmp x14, #0x10\n"
+      "mov x27, x12\n"
       "bgt 135f\n"
-      "cmp x13, #0xc\n"
-      "mov x27, x11\n"
+      "cmp x14, #0xc\n"
+      "mov x28, x12\n"
       "bgt 135f\n"
-      "cmp x13, #0x8\n"
-      "mov x28, x11\n"
+      "cmp x14, #0x8\n"
+      "mov x9, x12\n"
       "bgt 135f\n"
-      "cmp x13, #0x4\n"
-      "mov x9, x11\n"
+      "cmp x14, #0x4\n"
+      "mov x10, x12\n"
       "bgt 135f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "135:"  // Height 4: B setup done
-      "cbz x14, 136f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
+      "cbz x15, 136f\n"
+      "ldr q8, [x15, #0x0]\n"
+      "ldr q9, [x15, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q10, [x15, #0x20]\n"
+      "ldr q11, [x15, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x14, #0x40]\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q12, [x15, #0x40]\n"
+      "ldr q13, [x15, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
-      "add x14, x14, #0x60\n"
+      "add x15, x15, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -1816,204 +1816,204 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "b 152f\n"
       "136:"  // Height 4: no bias
       "tbz %x[flags], #0, 151f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "cmp x13, #0x18\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x14, #0x18\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 149f\n"
-      "tbz x13, #4, 140f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v12.4s }, [x12], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 138f\n"
-      "ld1 { v13.4s }, [x12], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 137f\n"
-      "ldr d20, [x12], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "mov x19, #0x58\n"
-      "ldr d4, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v20.s }[2], [x12]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "tbz x14, #4, 140f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 138f\n"
+      "ld1 { v13.4s }, [x13], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 137f\n"
+      "ldr d20, [x13], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d4, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v20.s }[2], [x13]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 148f\n"
       "137:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s20, [x12, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "mov x20, #0x50\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s20, [x13, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 148f\n"
       "138:"  // Height 4: Partial accumulate: partial_2_16
-      "tbz x13, #1, 139f\n"
-      "ldr d13, [x12], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v13.s }[2], [x12]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "tbz x14, #1, 139f\n"
+      "ldr d13, [x13], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 148f\n"
       "139:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s13, [x12, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "mov x20, #0x40\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s13, [x13, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 148f\n"
       "140:"  // Height 4: Partial accumulate: partial_8_0
-      "tbz x13, #3, 144f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x12], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 142f\n"
-      "ld1 { v11.4s }, [x12], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 141f\n"
-      "ldr d12, [x12], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v12.s }[2], [x12]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "tbz x14, #3, 144f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 142f\n"
+      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 141f\n"
+      "ldr d12, [x13], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 148f\n"
       "141:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s12, [x12, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "mov x20, #0x30\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s12, [x13, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 148f\n"
       "142:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x13, #1, 143f\n"
-      "ldr d11, [x12], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v11.s }[2], [x12]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "tbz x14, #1, 143f\n"
+      "ldr d11, [x13], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 148f\n"
       "143:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s11, [x12, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "mov x20, #0x20\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s11, [x13, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "b 148f\n"
       "144:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x13, #2, 146f\n"
-      "ld1 { v9.4s }, [x12], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 145f\n"
-      "ldr d10, [x12], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v10.s }[2], [x12]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x14, #2, 146f\n"
+      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 145f\n"
+      "ldr d10, [x13], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 148f\n"
       "145:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
-      "tbz x13, #0, 148f\n"
-      "ldr s10, [x12, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x20, #0x10\n"
+      "tbz x14, #0, 148f\n"
+      "ldr s10, [x13, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 148f\n"
       "146:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x13, #1, 147f\n"
-      "ldr d9, [x12], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x13, #0, 148f\n"
-      "ld1 { v9.s }[2], [x12]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x14, #1, 147f\n"
+      "ldr d9, [x13], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x14, #0, 148f\n"
+      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 148f\n"
       "147:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x12, #0x0]\n"
-      "ldr s14, [x22, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "ldr s9, [x13, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "148:"  // Height 4: Partial accumulate: Done
-      "sub x12, x12, x19\n"
+      "sub x13, x13, x20\n"
       "b 150f\n"
       "149:"  // Height 4: full accumulate
-      "ldr q9, [x12, #0x0]\n"
-      "ldr q10, [x12, #0x10]\n"
-      "ldr q11, [x12, #0x20]\n"
-      "ldr q12, [x12, #0x30]\n"
-      "ldr q13, [x12, #0x40]\n"
-      "ldr q20, [x12, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q21, [x21, #0x0]\n"
-      "ldr q22, [x21, #0x10]\n"
-      "ldr q23, [x21, #0x20]\n"
-      "ldr q24, [x21, #0x30]\n"
-      "ldr q25, [x21, #0x40]\n"
-      "ldr q4, [x21, #0x50]\n"
-      "ldr q26, [x20, #0x0]\n"
-      "ldr q27, [x20, #0x10]\n"
-      "ldr q28, [x20, #0x20]\n"
-      "ldr q29, [x20, #0x30]\n"
-      "ldr q30, [x20, #0x40]\n"
-      "ldr q31, [x20, #0x50]\n"
+      "ldr q9, [x13, #0x0]\n"
+      "ldr q10, [x13, #0x10]\n"
+      "ldr q11, [x13, #0x20]\n"
+      "ldr q12, [x13, #0x30]\n"
+      "ldr q13, [x13, #0x40]\n"
+      "ldr q20, [x13, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q21, [x22, #0x0]\n"
+      "ldr q22, [x22, #0x10]\n"
+      "ldr q23, [x22, #0x20]\n"
+      "ldr q24, [x22, #0x30]\n"
+      "ldr q25, [x22, #0x40]\n"
+      "ldr q4, [x22, #0x50]\n"
+      "ldr q26, [x21, #0x0]\n"
+      "ldr q27, [x21, #0x10]\n"
+      "ldr q28, [x21, #0x20]\n"
+      "ldr q29, [x21, #0x30]\n"
+      "ldr q30, [x21, #0x40]\n"
+      "ldr q31, [x21, #0x50]\n"
       "150:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -2066,133 +2066,133 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "152:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "153:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 154f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 155f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 155f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 155f\n"
       "154:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "155:"  // Height 4: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 158f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "cmp x24, #0x8\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "ld1 { v3.4s }, [x20], #0x10\n"
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "ld1 { v3.4s }, [x21], #0x10\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       "blt 157f\n"
       "156:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x8\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ld1 { v3.4s }, [x20], #0x10\n"
+      "ld1 { v3.4s }, [x21], #0x10\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
-      "add x11, x11, #0x20\n"
+      "ldr q4, [x10, #0x0]\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q5, [x10, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x9, #0x0]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q7, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
-      "add x27, x27, #0x20\n"
+      "ldr q5, [x28, #0x10]\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "add x26, x26, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x11, #0x0]\n"
+      "ldr q4, [x12, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec59  // bfmmla v25.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x10, #0x0]\n"
+      "ldr q6, [x11, #0x0]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q7, [x11, #0x10]\n"
       "bge 156b\n"
       "157:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "sub x24, x24, #0x4\n"
-      "add x11, x11, #0x20\n"
+      "sub x25, x25, #0x4\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "add x10, x10, #0x20\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
+      "ldr q4, [x10, #0x0]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
+      "ldr q5, [x10, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "add x9, x9, #0x20\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "add x28, x28, #0x20\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "add x27, x27, #0x20\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "add x26, x26, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
@@ -2202,63 +2202,63 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "158:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 161f\n"
-      "cbz x24, 161f\n"
-      "tbz x24, #1, 159f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "ldr d2, [x21], #0x8\n"
-      "ldr d3, [x20], #0x8\n"
-      "tbz x24, #0, 160f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
-      "ld1 { v2.s }[2], [x21]\n"
-      "ld1 { v3.s }[2], [x20]\n"
+      "cbz x25, 161f\n"
+      "cbz x25, 161f\n"
+      "tbz x25, #1, 159f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "ldr d2, [x22], #0x8\n"
+      "ldr d3, [x21], #0x8\n"
+      "tbz x25, #0, 160f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
+      "ld1 { v2.s }[2], [x22]\n"
+      "ld1 { v3.s }[2], [x21]\n"
       "b 160f\n"
       "159:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
-      "ldr s2, [x21, #0x0]\n"
-      "ldr s3, [x20, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
+      "ldr s2, [x22, #0x0]\n"
+      "ldr s3, [x21, #0x0]\n"
       "160:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "ldr q4, [x11, #0x0]\n"
-      "ldr q5, [x11, #0x10]\n"
+      "ldr q4, [x12, #0x0]\n"
+      "ldr q5, [x12, #0x10]\n"
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x11, #0x0]\n"
+      "ldr q7, [x11, #0x10]\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x9, #0x0]\n"
-      "add x11, x11, #0x20\n"
+      "ldr q4, [x10, #0x0]\n"
+      "add x12, x12, #0x20\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x9, #0x10]\n"
-      "add x10, x10, #0x20\n"
+      "ldr q5, [x10, #0x10]\n"
+      "add x11, x11, #0x20\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x28, #0x0]\n"
-      "add x9, x9, #0x20\n"
+      "ldr q6, [x9, #0x0]\n"
+      "add x10, x10, #0x20\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x28, x28, #0x20\n"
+      "ldr q7, [x9, #0x10]\n"
+      "add x9, x9, #0x20\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
-      "ldr q4, [x27, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
-      "ldr q5, [x27, #0x10]\n"
-      "add x27, x27, #0x20\n"
+      "ldr q5, [x28, #0x10]\n"
+      "add x28, x28, #0x20\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q6, [x26, #0x0]\n"
+      "ldr q6, [x27, #0x0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
-      "ldr q7, [x26, #0x10]\n"
-      "add x26, x26, #0x20\n"
+      "ldr q7, [x27, #0x10]\n"
+      "add x27, x27, #0x20\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
@@ -2268,17 +2268,17 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "161:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 153b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
       "uzp2 v10.2d, v10.2d, v16.2d\n"
@@ -2301,10 +2301,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v30.2d, v25.2d, v31.2d\n"
       "uzp2 v25.2d, v25.2d, v31.2d\n"
       "tbz %x[flags], #1, 162f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v4.4s, v4.4s, v1.4s\n"
       "fmin v14.4s, v14.4s, v1.4s\n"
       "fmin v15.4s, v15.4s, v1.4s\n"
@@ -2354,206 +2354,206 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 (
       "fmax v24.4s, v24.4s, v0.4s\n"
       "fmax v25.4s, v25.4s, v0.4s\n"
       "162:"  // Height 4: No activation
-      "cmp x13, #0x18\n"
+      "cmp x14, #0x18\n"
       "bge 175f\n"
-      "tbz x13, #4, 166f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v16.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v27.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
-      "st1 { v21.4s }, [x20], #0x10\n"
-      "st1 { v22.4s }, [x20], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 164f\n"
-      "st1 { v17.4s }, [x12], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
-      "st1 { v29.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 163f\n"
-      "str d18, [x12], #0x8\n"
-      "str d13, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v18.s }[2], [x12]\n"
-      "st1 { v13.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x14, #4, 166f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v16.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v27.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v21.4s }, [x21], #0x10\n"
+      "st1 { v22.4s }, [x21], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 164f\n"
+      "st1 { v17.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
+      "st1 { v29.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 163f\n"
+      "str d18, [x13], #0x8\n"
+      "str d13, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v18.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 174f\n"
       "163:"  // Height 4: Partial direct writeback: partial_1_20
-      "tbz x13, #0, 174f\n"
-      "str s18, [x12, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x14, #0, 174f\n"
+      "str s18, [x13, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 174f\n"
       "164:"  // Height 4: Partial direct writeback: partial_2_16
-      "tbz x13, #1, 165f\n"
-      "str d17, [x12], #0x8\n"
-      "str d12, [x22], #0x8\n"
-      "str d29, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v17.s }[2], [x12]\n"
-      "st1 { v12.s }[2], [x22]\n"
-      "st1 { v29.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x14, #1, 165f\n"
+      "str d17, [x13], #0x8\n"
+      "str d12, [x23], #0x8\n"
+      "str d29, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v17.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x23]\n"
+      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 174f\n"
       "165:"  // Height 4: Partial direct writeback: partial_1_16
-      "tbz x13, #0, 174f\n"
-      "str s17, [x12, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
-      "str s29, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "tbz x14, #0, 174f\n"
+      "str s17, [x13, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
+      "str s29, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "b 174f\n"
       "166:"  // Height 4: Partial direct writeback: partial_8_0
-      "tbz x13, #3, 170f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v14.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
-      "st1 { v21.4s }, [x20], #0x10\n"
-      "tbz x13, #2, 168f\n"
-      "st1 { v15.4s }, [x12], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v27.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 167f\n"
-      "str d16, [x12], #0x8\n"
-      "str d11, [x22], #0x8\n"
-      "str d28, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v16.s }[2], [x12]\n"
-      "st1 { v11.s }[2], [x22]\n"
-      "st1 { v28.s }[2], [x21]\n"
-      "st1 { v23.s }[2], [x20]\n"
+      "tbz x14, #3, 170f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v14.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v21.4s }, [x21], #0x10\n"
+      "tbz x14, #2, 168f\n"
+      "st1 { v15.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v27.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 167f\n"
+      "str d16, [x13], #0x8\n"
+      "str d11, [x23], #0x8\n"
+      "str d28, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v16.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x23]\n"
+      "st1 { v28.s }[2], [x22]\n"
+      "st1 { v23.s }[2], [x21]\n"
       "b 174f\n"
       "167:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x13, #0, 174f\n"
-      "str s16, [x12, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
-      "str s28, [x21, #0x0]\n"
-      "str s23, [x20, #0x0]\n"
+      "tbz x14, #0, 174f\n"
+      "str s16, [x13, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
+      "str s28, [x22, #0x0]\n"
+      "str s23, [x21, #0x0]\n"
       "b 174f\n"
       "168:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x13, #1, 169f\n"
-      "str d15, [x12], #0x8\n"
-      "str d10, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d22, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v15.s }[2], [x12]\n"
-      "st1 { v10.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
-      "st1 { v22.s }[2], [x20]\n"
+      "tbz x14, #1, 169f\n"
+      "str d15, [x13], #0x8\n"
+      "str d10, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d22, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v15.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
+      "st1 { v22.s }[2], [x21]\n"
       "b 174f\n"
       "169:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x13, #0, 174f\n"
-      "str s15, [x12, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
-      "str s22, [x20, #0x0]\n"
+      "tbz x14, #0, 174f\n"
+      "str s15, [x13, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
+      "str s22, [x21, #0x0]\n"
       "b 174f\n"
       "170:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x13, #2, 172f\n"
-      "st1 { v4.4s }, [x12], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
-      "tbz x13, #1, 171f\n"
-      "str d14, [x12], #0x8\n"
-      "str d9, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d21, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v14.s }[2], [x12]\n"
-      "st1 { v9.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
-      "st1 { v21.s }[2], [x20]\n"
+      "tbz x14, #2, 172f\n"
+      "st1 { v4.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
+      "tbz x14, #1, 171f\n"
+      "str d14, [x13], #0x8\n"
+      "str d9, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d21, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v14.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
+      "st1 { v21.s }[2], [x21]\n"
       "b 174f\n"
       "171:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x13, #0, 174f\n"
-      "str s14, [x12, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
-      "str s21, [x20, #0x0]\n"
+      "tbz x14, #0, 174f\n"
+      "str s14, [x13, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
+      "str s21, [x21, #0x0]\n"
       "b 174f\n"
       "172:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x13, #1, 173f\n"
-      "str d4, [x12], #0x8\n"
-      "str d8, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d20, [x20], #0x8\n"
-      "tbz x13, #0, 174f\n"
-      "st1 { v4.s }[2], [x12]\n"
-      "st1 { v8.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v20.s }[2], [x20]\n"
+      "tbz x14, #1, 173f\n"
+      "str d4, [x13], #0x8\n"
+      "str d8, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d20, [x21], #0x8\n"
+      "tbz x14, #0, 174f\n"
+      "st1 { v4.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v20.s }[2], [x21]\n"
       "b 174f\n"
       "173:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s4, [x12, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s20, [x20, #0x0]\n"
+      "str s4, [x13, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s20, [x21, #0x0]\n"
       "174:"  // Height 4: Partial direct writeback: Done
       "b 176f\n"
       "175:"  // Height 4: Full writeback
-      "str q4, [x12, #0x0]\n"
-      "str q14, [x12, #0x10]\n"
-      "str q15, [x12, #0x20]\n"
-      "str q16, [x12, #0x30]\n"
-      "str q17, [x12, #0x40]\n"
-      "str q18, [x12, #0x50]\n"
-      "add x12, x12, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
-      "str q19, [x21, #0x0]\n"
-      "str q26, [x21, #0x10]\n"
-      "str q27, [x21, #0x20]\n"
-      "str q28, [x21, #0x30]\n"
-      "str q29, [x21, #0x40]\n"
-      "str q30, [x21, #0x50]\n"
-      "str q20, [x20, #0x0]\n"
-      "str q21, [x20, #0x10]\n"
-      "str q22, [x20, #0x20]\n"
-      "str q23, [x20, #0x30]\n"
-      "str q24, [x20, #0x40]\n"
-      "str q25, [x20, #0x50]\n"
+      "str q4, [x13, #0x0]\n"
+      "str q14, [x13, #0x10]\n"
+      "str q15, [x13, #0x20]\n"
+      "str q16, [x13, #0x30]\n"
+      "str q17, [x13, #0x40]\n"
+      "str q18, [x13, #0x50]\n"
+      "add x13, x13, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
+      "str q19, [x22, #0x0]\n"
+      "str q26, [x22, #0x10]\n"
+      "str q27, [x22, #0x20]\n"
+      "str q28, [x22, #0x30]\n"
+      "str q29, [x22, #0x40]\n"
+      "str q30, [x22, #0x50]\n"
+      "str q20, [x21, #0x0]\n"
+      "str q21, [x21, #0x10]\n"
+      "str q22, [x21, #0x20]\n"
+      "str q23, [x21, #0x30]\n"
+      "str q24, [x21, #0x40]\n"
+      "str q25, [x21, #0x50]\n"
       "176:"  // Height 4: Writeback done
-      "subs x13, x13, #0x18\n"
+      "subs x14, x14, #0x18\n"
       "bgt 134b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 178f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 177f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "177:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "178:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp
index 967396c377..2458d6a035 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -52,34 +52,34 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
 
     __asm__ __volatile__(
       "1:"  // Height loop
-      "ldr x24, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x23, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x22, %x[Apanel]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x23, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x21, x24, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x23, #0x8\n"
-      "mov %x[Apanel], x22\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x22, x25, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x24, #0x8\n"
+      "mov %x[Apanel], x23\n"
       "bgt 3f\n"
-      "cmp x23, #0x4\n"
-      "mov x20, x24\n"
+      "cmp x24, #0x4\n"
+      "mov x21, x25\n"
       "bgt 3f\n"
-      "mov x21, x24\n"
+      "mov x22, x25\n"
       "3:"  // B setup done
       "ldr q0, [%x[Apanel], #0x0]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.16b, #0x0\n"
-      "ldr q4, [x24, #0x0]\n"
-      "ldr q5, [x21, #0x0]\n"
+      "ldr q4, [x25, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
       "movi v9.16b, #0x0\n"
-      "ldr q6, [x20, #0x0]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "movi v12.16b, #0x0\n"
@@ -109,38 +109,38 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f40f088  // bfdot v8.4s, v4.8h, v0.h[0]\n"
       ".inst 0x4f60f08b  // bfdot v11.4s, v4.8h, v0.h[1]\n"
       ".inst 0x4f40f88e  // bfdot v14.4s, v4.8h, v0.h[2]\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x4f60f891  // bfdot v17.4s, v4.8h, v0.h[3]\n"
       ".inst 0x4f41f094  // bfdot v20.4s, v4.8h, v1.h[0]\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x4f61f097  // bfdot v23.4s, v4.8h, v1.h[1]\n"
       ".inst 0x4f41f89a  // bfdot v26.4s, v4.8h, v1.h[2]\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x4f61f89d  // bfdot v29.4s, v4.8h, v1.h[3]\n"
-      "ldr q4, [x24, #0x10]\n"
+      "ldr q4, [x25, #0x10]\n"
       ".inst 0x4f40f0a9  // bfdot v9.4s, v5.8h, v0.h[0]\n"
       ".inst 0x4f60f0ac  // bfdot v12.4s, v5.8h, v0.h[1]\n"
       ".inst 0x4f40f8af  // bfdot v15.4s, v5.8h, v0.h[2]\n"
-      "add x24, x24, #0x20\n"
+      "add x25, x25, #0x20\n"
       ".inst 0x4f60f8b2  // bfdot v18.4s, v5.8h, v0.h[3]\n"
       ".inst 0x4f41f0b5  // bfdot v21.4s, v5.8h, v1.h[0]\n"
       ".inst 0x4f61f0b8  // bfdot v24.4s, v5.8h, v1.h[1]\n"
       ".inst 0x4f41f8bb  // bfdot v27.4s, v5.8h, v1.h[2]\n"
       ".inst 0x4f61f8be  // bfdot v30.4s, v5.8h, v1.h[3]\n"
-      "ldr q5, [x21, #0x10]\n"
+      "ldr q5, [x22, #0x10]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f60f0cd  // bfdot v13.4s, v6.8h, v0.h[1]\n"
       ".inst 0x4f40f8d0  // bfdot v16.4s, v6.8h, v0.h[2]\n"
-      "add x21, x21, #0x20\n"
+      "add x22, x22, #0x20\n"
       ".inst 0x4f60f8d3  // bfdot v19.4s, v6.8h, v0.h[3]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x4f41f0d6  // bfdot v22.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f61f0d9  // bfdot v25.4s, v6.8h, v1.h[1]\n"
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
-      "ldr q6, [x20, #0x10]\n"
+      "ldr q6, [x21, #0x10]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "add x20, x20, #0x20\n"
+      "add x21, x21, #0x20\n"
       ".inst 0x4f42f088  // bfdot v8.4s, v4.8h, v2.h[0]\n"
       ".inst 0x4f62f08b  // bfdot v11.4s, v4.8h, v2.h[1]\n"
       ".inst 0x4f42f88e  // bfdot v14.4s, v4.8h, v2.h[2]\n"
@@ -149,7 +149,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f63f097  // bfdot v23.4s, v4.8h, v3.h[1]\n"
       ".inst 0x4f43f89a  // bfdot v26.4s, v4.8h, v3.h[2]\n"
       ".inst 0x4f63f89d  // bfdot v29.4s, v4.8h, v3.h[3]\n"
-      "ldr q4, [x24, #0x0]\n"
+      "ldr q4, [x25, #0x0]\n"
       ".inst 0x4f42f0a9  // bfdot v9.4s, v5.8h, v2.h[0]\n"
       ".inst 0x4f62f0ac  // bfdot v12.4s, v5.8h, v2.h[1]\n"
       ".inst 0x4f42f8af  // bfdot v15.4s, v5.8h, v2.h[2]\n"
@@ -158,7 +158,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f63f0b8  // bfdot v24.4s, v5.8h, v3.h[1]\n"
       ".inst 0x4f43f8bb  // bfdot v27.4s, v5.8h, v3.h[2]\n"
       ".inst 0x4f63f8be  // bfdot v30.4s, v5.8h, v3.h[3]\n"
-      "ldr q5, [x21, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
       ".inst 0x4f42f0ca  // bfdot v10.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f62f0cd  // bfdot v13.4s, v6.8h, v2.h[1]\n"
       ".inst 0x4f42f8d0  // bfdot v16.4s, v6.8h, v2.h[2]\n"
@@ -167,7 +167,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f63f0d9  // bfdot v25.4s, v6.8h, v3.h[1]\n"
       ".inst 0x4f43f8dc  // bfdot v28.4s, v6.8h, v3.h[2]\n"
       ".inst 0x4f63f8df  // bfdot v31.4s, v6.8h, v3.h[3]\n"
-      "ldr q6, [x20, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 4b\n"
       "5:"  // main loop skip
       ".inst 0x4f40f088  // bfdot v8.4s, v4.8h, v0.h[0]\n"
@@ -175,13 +175,13 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       "add %x[Apanel], %x[Apanel], #0x20\n"
       ".inst 0x4f40f88e  // bfdot v14.4s, v4.8h, v0.h[2]\n"
       ".inst 0x4f60f891  // bfdot v17.4s, v4.8h, v0.h[3]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f41f094  // bfdot v20.4s, v4.8h, v1.h[0]\n"
       ".inst 0x4f61f097  // bfdot v23.4s, v4.8h, v1.h[1]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f41f89a  // bfdot v26.4s, v4.8h, v1.h[2]\n"
       ".inst 0x4f61f89d  // bfdot v29.4s, v4.8h, v1.h[3]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f40f0a9  // bfdot v9.4s, v5.8h, v0.h[0]\n"
       ".inst 0x4f60f0ac  // bfdot v12.4s, v5.8h, v0.h[1]\n"
       ".inst 0x4f40f8af  // bfdot v15.4s, v5.8h, v0.h[2]\n"
@@ -198,14 +198,14 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f61f0d9  // bfdot v25.4s, v6.8h, v1.h[1]\n"
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
-      "cbz x19, 6f\n"
+      "cbz x20, 6f\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ldr q7, [x24, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q7, [x25, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x4f40f0e8  // bfdot v8.4s, v7.8h, v0.h[0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       ".inst 0x4f60f0eb  // bfdot v11.4s, v7.8h, v0.h[1]\n"
       ".inst 0x4f40f8ee  // bfdot v14.4s, v7.8h, v0.h[2]\n"
       ".inst 0x4f60f8f1  // bfdot v17.4s, v7.8h, v0.h[3]\n"
@@ -230,7 +230,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       ".inst 0x4f41f8bc  // bfdot v28.4s, v5.8h, v1.h[2]\n"
       ".inst 0x4f61f8bf  // bfdot v31.4s, v5.8h, v1.h[3]\n"
       "6:"  // multiply loop done
-      "subs x23, x23, #0xc\n"
+      "subs x24, x24, #0xc\n"
       "str q8, [%x[Cpanel], #0x0]\n"
       "str q9, [%x[Cpanel], #0x10]\n"
       "str q10, [%x[Cpanel], #0x20]\n"
@@ -261,7 +261,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp
index 509f2afa09..47991114af 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -52,37 +52,37 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
 
     __asm__ __volatile__(
       "1:"  // Height loop
-      "ldr x24, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x23, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x22, %x[Apanel]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x23, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x21, x24, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x23, #0x8\n"
-      "mov %x[Apanel], x22\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x22, x25, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x24, #0x8\n"
+      "mov %x[Apanel], x23\n"
       "bgt 3f\n"
-      "cmp x23, #0x4\n"
-      "mov x20, x24\n"
+      "cmp x24, #0x4\n"
+      "mov x21, x25\n"
       "bgt 3f\n"
-      "mov x21, x24\n"
+      "mov x22, x25\n"
       "3:"  // B setup done
-      "ldr q4, [x24, #0x0]\n"
+      "ldr q4, [x25, #0x0]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       "movi v8.16b, #0x0\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q5, [x24, #0x10]\n"
+      "ldr q5, [x25, #0x10]\n"
       "movi v9.16b, #0x0\n"
       "ldr q2, [%x[Apanel], #0x20]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
-      "add x24, x24, #0x20\n"
+      "add x25, x25, #0x20\n"
       "movi v12.16b, #0x0\n"
       "movi v13.16b, #0x0\n"
       "add %x[Apanel], %x[Apanel], #0x30\n"
@@ -107,30 +107,30 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       "blt 5f\n"
       "4:"  // main loop head
       "ldr q3, [%x[Apanel], #0x0]\n"
-      "ldr q6, [x21, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q7, [x21, #0x10]\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
-      "ldr q4, [x20, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldr q5, [x20, #0x10]\n"
+      "ldr q5, [x21, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x24, #0x0]\n"
+      "ldr q6, [x25, #0x0]\n"
       ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x24, #0x10]\n"
+      "ldr q7, [x25, #0x10]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
       "ldr q0, [%x[Apanel], #0x10]\n"
@@ -141,32 +141,32 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       "ldr q2, [%x[Apanel], #0x30]\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
-      "ldr q4, [x21, #0x20]\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
       "ldr q3, [%x[Apanel], #0x40]\n"
-      "ldr q5, [x21, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
-      "add x21, x21, #0x40\n"
+      "add x22, x22, #0x40\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x20, #0x20]\n"
+      "ldr q6, [x21, #0x20]\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x20, #0x30]\n"
+      "ldr q7, [x21, #0x30]\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
-      "add x20, x20, #0x40\n"
+      "add x21, x21, #0x40\n"
       ".inst 0x6e44ec55  // bfmmla v21.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7b  // bfmmla v27.4s, v3.8h, v4.8h\n"
-      "ldr q4, [x24, #0x20]\n"
+      "ldr q4, [x25, #0x20]\n"
       ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
-      "ldr q5, [x24, #0x30]\n"
+      "ldr q5, [x25, #0x30]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       "ldr q0, [%x[Apanel], #0x50]\n"
@@ -179,13 +179,13 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       ".inst 0x6e46ec7c  // bfmmla v28.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7f  // bfmmla v31.4s, v3.8h, v7.8h\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "add x24, x24, #0x40\n"
+      "add x25, x25, #0x40\n"
       "bge 4b\n"
       "5:"  // main loop skip
       "ldr q3, [%x[Apanel], #0x0]\n"
-      "ldr q6, [x21, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q7, [x21, #0x10]\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
@@ -193,16 +193,16 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
-      "ldr q4, [x20, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldr q5, [x20, #0x10]\n"
+      "ldr q5, [x21, #0x10]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "add x21, x21, #0x20\n"
+      "add x22, x22, #0x20\n"
       ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
-      "add x20, x20, #0x20\n"
+      "add x21, x21, #0x20\n"
       ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
@@ -214,26 +214,26 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
-      "cbz x19, 6f\n"
-      "ldr q6, [x24, #0x0]\n"
+      "cbz x20, 6f\n"
+      "ldr q6, [x25, #0x0]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q7, [x24, #0x10]\n"
+      "ldr q7, [x25, #0x10]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q2, [%x[Apanel], #0x20]\n"
       "ldr q3, [%x[Apanel], #0x30]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x21, #0x10]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x22, #0x10]\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x20, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x20, #0x10]\n"
+      "ldr q7, [x21, #0x10]\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
@@ -251,7 +251,7 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       ".inst 0x6e46ec7c  // bfmmla v28.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7f  // bfmmla v31.4s, v3.8h, v7.8h\n"
       "6:"  // multiply loop done
-      "subs x23, x23, #0xc\n"
+      "subs x24, x24, #0xc\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -306,7 +306,7 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp
index 19836f2e9d..36bfccf52f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
 
@@ -51,33 +51,33 @@ void a64_ffinterleaved_fp16_mla_8x24(
 
     __asm__ __volatile__(
       "1:"  // Height loop
-      "ldr x24, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x23, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x22, %x[Apanel]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x23, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x21, x24, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x23, #0x10\n"
-      "mov %x[Apanel], x22\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x22, x25, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x24, #0x10\n"
+      "mov %x[Apanel], x23\n"
       "bgt 3f\n"
-      "cmp x23, #0x8\n"
-      "mov x20, x24\n"
+      "cmp x24, #0x8\n"
+      "mov x21, x25\n"
       "bgt 3f\n"
-      "mov x21, x24\n"
+      "mov x22, x25\n"
       "3:"  // B setup done
       "ldr q0, [%x[Apanel], #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       "movi v8.16b, #0x0\n"
-      "ldr q3, [x21, #0x0]\n"
-      "ldr q4, [x20, #0x0]\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
       "movi v9.16b, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "movi v12.16b, #0x0\n"
@@ -103,35 +103,35 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "blt 5f\n"
       "4:"  // main loop head
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q5, [x24, #0x10]\n"
+      "ldr q5, [x25, #0x10]\n"
       "fmla v8.8h, v2.8h, v0.h[0]\n"
-      "ldr q6, [x21, #0x10]\n"
-      "ldr q7, [x20, #0x10]\n"
+      "ldr q6, [x22, #0x10]\n"
+      "ldr q7, [x21, #0x10]\n"
       "fmla v11.8h, v2.8h, v0.h[1]\n"
       "fmla v14.8h, v2.8h, v0.h[2]\n"
       "fmla v17.8h, v2.8h, v0.h[3]\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla v20.8h, v2.8h, v0.h[4]\n"
       "fmla v23.8h, v2.8h, v0.h[5]\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla v26.8h, v2.8h, v0.h[6]\n"
       "fmla v29.8h, v2.8h, v0.h[7]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla v9.8h, v3.8h, v0.h[0]\n"
       "fmla v12.8h, v3.8h, v0.h[1]\n"
-      "add x24, x24, #0x20\n"
-      "ldr q2, [x24, #0x0]\n"
+      "add x25, x25, #0x20\n"
+      "ldr q2, [x25, #0x0]\n"
       "fmla v15.8h, v3.8h, v0.h[2]\n"
       "fmla v18.8h, v3.8h, v0.h[3]\n"
       "fmla v21.8h, v3.8h, v0.h[4]\n"
       "fmla v24.8h, v3.8h, v0.h[5]\n"
-      "add x21, x21, #0x20\n"
+      "add x22, x22, #0x20\n"
       "fmla v27.8h, v3.8h, v0.h[6]\n"
       "fmla v30.8h, v3.8h, v0.h[7]\n"
-      "ldr q3, [x21, #0x0]\n"
+      "ldr q3, [x22, #0x0]\n"
       "fmla v10.8h, v4.8h, v0.h[0]\n"
       "fmla v13.8h, v4.8h, v0.h[1]\n"
-      "add x20, x20, #0x20\n"
+      "add x21, x21, #0x20\n"
       "fmla v16.8h, v4.8h, v0.h[2]\n"
       "fmla v19.8h, v4.8h, v0.h[3]\n"
       "fmla v22.8h, v4.8h, v0.h[4]\n"
@@ -139,7 +139,7 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "fmla v28.8h, v4.8h, v0.h[6]\n"
       "fmla v31.8h, v4.8h, v0.h[7]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
-      "ldr q4, [x20, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
       "fmla v8.8h, v5.8h, v1.h[0]\n"
       "fmla v11.8h, v5.8h, v1.h[1]\n"
       "fmla v14.8h, v5.8h, v1.h[2]\n"
@@ -171,13 +171,13 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla v14.8h, v2.8h, v0.h[2]\n"
       "fmla v17.8h, v2.8h, v0.h[3]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v20.8h, v2.8h, v0.h[4]\n"
       "fmla v23.8h, v2.8h, v0.h[5]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v26.8h, v2.8h, v0.h[6]\n"
       "fmla v29.8h, v2.8h, v0.h[7]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v9.8h, v3.8h, v0.h[0]\n"
       "fmla v12.8h, v3.8h, v0.h[1]\n"
       "fmla v15.8h, v3.8h, v0.h[2]\n"
@@ -194,12 +194,12 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "fmla v25.8h, v4.8h, v0.h[5]\n"
       "fmla v28.8h, v4.8h, v0.h[6]\n"
       "fmla v31.8h, v4.8h, v0.h[7]\n"
-      "cbz x19, 6f\n"
+      "cbz x20, 6f\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
-      "ldr q5, [x24, #0x0]\n"
+      "ldr q5, [x25, #0x0]\n"
       "fmla v8.8h, v5.8h, v0.h[0]\n"
-      "ldr q6, [x21, #0x0]\n"
-      "ldr q7, [x20, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
+      "ldr q7, [x21, #0x0]\n"
       "fmla v11.8h, v5.8h, v0.h[1]\n"
       "fmla v14.8h, v5.8h, v0.h[2]\n"
       "fmla v17.8h, v5.8h, v0.h[3]\n"
@@ -225,7 +225,7 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "fmla v28.8h, v7.8h, v0.h[6]\n"
       "fmla v31.8h, v7.8h, v0.h[7]\n"
       "6:"  // multiply loop done
-      "subs x23, x23, #0x18\n"
+      "subs x24, x24, #0x18\n"
       "str q8, [%x[Cpanel], #0x0]\n"
       "str q9, [%x[Cpanel], #0x10]\n"
       "str q10, [%x[Cpanel], #0x20]\n"
@@ -256,7 +256,7 @@ void a64_ffinterleaved_fp16_mla_8x24(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp
index bf804b5f43..ec99d64f4a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -51,34 +51,34 @@ void a64_ffinterleaved_fp32_mla_8x12(
 
     __asm__ __volatile__(
       "1:"  // Height loop
-      "ldr x24, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x23, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x22, %x[Apanel]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x23, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x21, x24, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "cmp x23, #0x8\n"
-      "mov %x[Apanel], x22\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x22, x25, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "cmp x24, #0x8\n"
+      "mov %x[Apanel], x23\n"
       "bgt 3f\n"
-      "cmp x23, #0x4\n"
-      "mov x20, x24\n"
+      "cmp x24, #0x4\n"
+      "mov x21, x25\n"
       "bgt 3f\n"
-      "mov x21, x24\n"
+      "mov x22, x25\n"
       "3:"  // B setup done
       "ldr q0, [%x[Apanel], #0x0]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.16b, #0x0\n"
-      "ldr q4, [x24, #0x0]\n"
-      "ldr q5, [x21, #0x0]\n"
+      "ldr q4, [x25, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
       "movi v9.16b, #0x0\n"
-      "ldr q6, [x20, #0x0]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x4\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x4\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "movi v12.16b, #0x0\n"
@@ -106,17 +106,17 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "ldr q2, [%x[Apanel], #0x20]\n"
       "ldr q3, [%x[Apanel], #0x30]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q7, [x24, #0x10]\n"
+      "ldr q7, [x25, #0x10]\n"
       "fmla v11.4s, v4.4s, v0.s[1]\n"
       "fmla v14.4s, v4.4s, v0.s[2]\n"
       "fmla v17.4s, v4.4s, v0.s[3]\n"
       "fmla v20.4s, v4.4s, v1.s[0]\n"
-      "sub x19, x19, #0x4\n"
+      "sub x20, x20, #0x4\n"
       "fmla v23.4s, v4.4s, v1.s[1]\n"
       "fmla v26.4s, v4.4s, v1.s[2]\n"
-      "cmp x19, #0x4\n"
+      "cmp x20, #0x4\n"
       "fmla v29.4s, v4.4s, v1.s[3]\n"
-      "ldr q4, [x21, #0x10]\n"
+      "ldr q4, [x22, #0x10]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
       "fmla v12.4s, v5.4s, v0.s[1]\n"
       "fmla v15.4s, v5.4s, v0.s[2]\n"
@@ -125,7 +125,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v24.4s, v5.4s, v1.s[1]\n"
       "fmla v27.4s, v5.4s, v1.s[2]\n"
       "fmla v30.4s, v5.4s, v1.s[3]\n"
-      "ldr q5, [x20, #0x10]\n"
+      "ldr q5, [x21, #0x10]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v13.4s, v6.4s, v0.s[1]\n"
       "fmla v16.4s, v6.4s, v0.s[2]\n"
@@ -136,7 +136,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v28.4s, v6.4s, v1.s[2]\n"
       "fmla v31.4s, v6.4s, v1.s[3]\n"
       "ldr q1, [%x[Apanel], #0x50]\n"
-      "ldr q6, [x24, #0x20]\n"
+      "ldr q6, [x25, #0x20]\n"
       "fmla v8.4s, v7.4s, v2.s[0]\n"
       "fmla v11.4s, v7.4s, v2.s[1]\n"
       "fmla v14.4s, v7.4s, v2.s[2]\n"
@@ -145,7 +145,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "fmla v26.4s, v7.4s, v3.s[2]\n"
       "fmla v29.4s, v7.4s, v3.s[3]\n"
-      "ldr q7, [x21, #0x20]\n"
+      "ldr q7, [x22, #0x20]\n"
       "fmla v9.4s, v4.4s, v2.s[0]\n"
       "fmla v12.4s, v4.4s, v2.s[1]\n"
       "fmla v15.4s, v4.4s, v2.s[2]\n"
@@ -154,7 +154,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v24.4s, v4.4s, v3.s[1]\n"
       "fmla v27.4s, v4.4s, v3.s[2]\n"
       "fmla v30.4s, v4.4s, v3.s[3]\n"
-      "ldr q4, [x20, #0x20]\n"
+      "ldr q4, [x21, #0x20]\n"
       "fmla v10.4s, v5.4s, v2.s[0]\n"
       "fmla v13.4s, v5.4s, v2.s[1]\n"
       "fmla v16.4s, v5.4s, v2.s[2]\n"
@@ -165,7 +165,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v28.4s, v5.4s, v3.s[2]\n"
       "fmla v31.4s, v5.4s, v3.s[3]\n"
       "ldr q3, [%x[Apanel], #0x70]\n"
-      "ldr q5, [x24, #0x30]\n"
+      "ldr q5, [x25, #0x30]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v11.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v0.s[2]\n"
@@ -173,23 +173,23 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "add %x[Apanel], %x[Apanel], #0x80\n"
       "fmla v20.4s, v6.4s, v1.s[0]\n"
       "fmla v23.4s, v6.4s, v1.s[1]\n"
-      "add x24, x24, #0x40\n"
+      "add x25, x25, #0x40\n"
       "fmla v26.4s, v6.4s, v1.s[2]\n"
       "fmla v29.4s, v6.4s, v1.s[3]\n"
-      "ldr q6, [x21, #0x30]\n"
+      "ldr q6, [x22, #0x30]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v12.4s, v7.4s, v0.s[1]\n"
-      "add x21, x21, #0x40\n"
+      "add x22, x22, #0x40\n"
       "fmla v15.4s, v7.4s, v0.s[2]\n"
       "fmla v18.4s, v7.4s, v0.s[3]\n"
       "fmla v21.4s, v7.4s, v1.s[0]\n"
       "fmla v24.4s, v7.4s, v1.s[1]\n"
       "fmla v27.4s, v7.4s, v1.s[2]\n"
       "fmla v30.4s, v7.4s, v1.s[3]\n"
-      "ldr q7, [x20, #0x30]\n"
+      "ldr q7, [x21, #0x30]\n"
       "fmla v10.4s, v4.4s, v0.s[0]\n"
       "fmla v13.4s, v4.4s, v0.s[1]\n"
-      "add x20, x20, #0x40\n"
+      "add x21, x21, #0x40\n"
       "fmla v16.4s, v4.4s, v0.s[2]\n"
       "fmla v19.4s, v4.4s, v0.s[3]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
@@ -198,7 +198,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v28.4s, v4.4s, v1.s[2]\n"
       "fmla v31.4s, v4.4s, v1.s[3]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q4, [x24, #0x0]\n"
+      "ldr q4, [x25, #0x0]\n"
       "fmla v8.4s, v5.4s, v2.s[0]\n"
       "fmla v11.4s, v5.4s, v2.s[1]\n"
       "fmla v14.4s, v5.4s, v2.s[2]\n"
@@ -207,7 +207,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v23.4s, v5.4s, v3.s[1]\n"
       "fmla v26.4s, v5.4s, v3.s[2]\n"
       "fmla v29.4s, v5.4s, v3.s[3]\n"
-      "ldr q5, [x21, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
       "fmla v9.4s, v6.4s, v2.s[0]\n"
       "fmla v12.4s, v6.4s, v2.s[1]\n"
       "fmla v15.4s, v6.4s, v2.s[2]\n"
@@ -216,7 +216,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v24.4s, v6.4s, v3.s[1]\n"
       "fmla v27.4s, v6.4s, v3.s[2]\n"
       "fmla v30.4s, v6.4s, v3.s[3]\n"
-      "ldr q6, [x20, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "fmla v10.4s, v7.4s, v2.s[0]\n"
       "fmla v13.4s, v7.4s, v2.s[1]\n"
       "fmla v16.4s, v7.4s, v2.s[2]\n"
@@ -232,13 +232,13 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla v14.4s, v4.4s, v0.s[2]\n"
       "fmla v17.4s, v4.4s, v0.s[3]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v20.4s, v4.4s, v1.s[0]\n"
       "fmla v23.4s, v4.4s, v1.s[1]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v26.4s, v4.4s, v1.s[2]\n"
       "fmla v29.4s, v4.4s, v1.s[3]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
       "fmla v12.4s, v5.4s, v0.s[1]\n"
       "fmla v15.4s, v5.4s, v0.s[2]\n"
@@ -255,15 +255,15 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v25.4s, v6.4s, v1.s[1]\n"
       "fmla v28.4s, v6.4s, v1.s[2]\n"
       "fmla v31.4s, v6.4s, v1.s[3]\n"
-      "cbz x19, 7f\n"
+      "cbz x20, 7f\n"
       "6:"  // odd loop
       "ldr q0, [%x[Apanel], #0x0]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "subs x19, x19, #0x1\n"
-      "ldr q7, [x24, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "subs x20, x20, #0x1\n"
+      "ldr q7, [x25, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "fmla v8.4s, v7.4s, v0.s[0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
       "fmla v14.4s, v7.4s, v0.s[2]\n"
       "fmla v17.4s, v7.4s, v0.s[3]\n"
@@ -271,13 +271,13 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla v23.4s, v7.4s, v1.s[1]\n"
       "fmla v26.4s, v7.4s, v1.s[2]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v29.4s, v7.4s, v1.s[3]\n"
       "fmla v9.4s, v4.4s, v0.s[0]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v12.4s, v4.4s, v0.s[1]\n"
       "fmla v15.4s, v4.4s, v0.s[2]\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v18.4s, v4.4s, v0.s[3]\n"
       "fmla v21.4s, v4.4s, v1.s[0]\n"
       "fmla v24.4s, v4.4s, v1.s[1]\n"
@@ -293,7 +293,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "fmla v31.4s, v5.4s, v1.s[3]\n"
       "bne 6b\n"
       "7:"  // multiply loop done
-      "subs x23, x23, #0xc\n"
+      "subs x24, x24, #0xc\n"
       "str q8, [%x[Cpanel], #0x0]\n"
       "str q9, [%x[Cpanel], #0x10]\n"
       "str q10, [%x[Cpanel], #0x20]\n"
@@ -324,7 +324,7 @@ void a64_ffinterleaved_fp32_mla_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp
index 27e08135b6..02d2434356 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -103,82 +103,82 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 71f\n"
       "beq 36f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x9, 3f\n"
-      "ldr q8, [x9, #0x0]\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "cbz x12, 3f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "b 14f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 13f\n"
       "cmp x11, #0x10\n"
       "bge 12f\n"
       "tbz x11, #3, 7f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 5f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 4f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 11f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 11f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 11f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x11, #1, 6f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 11f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 11f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 11f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x11, #2, 9f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 8f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 11f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 11f\n"
-      "ldr s9, [x28, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
       "b 11f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x11, #1, 10f\n"
-      "ldr d8, [x28], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v8.s }[2], [x28]\n"
+      "ld1 { v8.s }[2], [x9]\n"
       "b 11f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x9, #0x0]\n"
+      "mov x20, #0x0\n"
       "11:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 14f\n"
       "12:"  // Height 1: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
       "b 14f\n"
       "13:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -186,46 +186,42 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "14:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 17f\n"
       "16:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "17:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 20f\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 19f\n"
       "18:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x8\n"
-      ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x10\n"
-      ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
+      ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x4f60f0c8  // bfdot v8.4s, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x4f60f0e9  // bfdot v9.4s, v7.8h, v0.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f60f0ca  // bfdot v10.4s, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x80]\n"
       ".inst 0x4f60f0eb  // bfdot v11.4s, v7.8h, v0.h[1]\n"
@@ -242,27 +238,28 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4f60f8e9  // bfdot v9.4s, v7.8h, v0.h[3]\n"
       "ldr q7, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f60f8ca  // bfdot v10.4s, v6.8h, v0.h[3]\n"
-      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "add x10, x10, #0x100\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 18b\n"
       "19:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x4f60f0c8  // bfdot v8.4s, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f60f0e9  // bfdot v9.4s, v7.8h, v0.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
       ".inst 0x4f60f0ca  // bfdot v10.4s, v6.8h, v0.h[1]\n"
@@ -281,221 +278,224 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4f60f8e9  // bfdot v9.4s, v7.8h, v0.h[3]\n"
       "ldr q7, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f60f8ca  // bfdot v10.4s, v6.8h, v0.h[3]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "20:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 24f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 24f\n"
+      "cmp x27, #0x2\n"
       "blt 22f\n"
       "21:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
+      "ldr s0, [x26], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
+      "sub x27, x27, #0x2\n"
       "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x2\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
+      "cmp x27, #0x2\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
+      "add x10, x10, #0x40\n"
       "bge 21b\n"
-      "cbz x26, 24f\n"
       "22:"  // Height 1: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
+      "cbz x27, 24f\n"
+      "ldr h0, [x26, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
-      "ldr q6, [x10, #0x20]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
+      "ldr q6, [x10, #0x20]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
+      "add x10, x10, #0x40\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 15b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
       "25:"  // Height 1: No activation
       "cmp x11, #0x10\n"
       "bge 34f\n"
       "tbz x11, #3, 29f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 27f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 26f\n"
-      "str d11, [x28], #0x8\n"
+      "str d11, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 33f\n"
       "26:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x11, #0, 33f\n"
-      "str s11, [x28, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
       "b 33f\n"
       "27:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x11, #1, 28f\n"
-      "str d10, [x28], #0x8\n"
+      "str d10, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 33f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x11, #0, 33f\n"
-      "str s10, [x28, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
       "b 33f\n"
       "29:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x11, #2, 31f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 30f\n"
-      "str d9, [x28], #0x8\n"
+      "str d9, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 33f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x11, #0, 33f\n"
-      "str s9, [x28, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
       "b 33f\n"
       "31:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x11, #1, 32f\n"
-      "str d8, [x28], #0x8\n"
+      "str d8, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 33f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "33:"  // Height 1: Partial direct writeback: Done
       "b 35f\n"
       "34:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "35:"  // Height 1: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 212f\n"
       "36:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "37:"  // Height 2: Column loop
-      "cbz x9, 38f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 38f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "add x12, x12, #0x40\n"
       "b 49f\n"
       "38:"  // Height 2: no bias
       "tbz %x[flags], #0, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "bge 47f\n"
       "tbz x11, #3, 42f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 40f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 39f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 46f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 46f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 46f\n"
       "40:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x11, #1, 41f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 46f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 46f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 46f\n"
       "42:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x11, #2, 44f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 43f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 46f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 46f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 46f\n"
       "44:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x11, #1, 45f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 46f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "46:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 49f\n"
       "47:"  // Height 2: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "b 49f\n"
       "48:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -507,58 +507,58 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "49:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "50:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 51f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 52f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 52f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 52f\n"
       "51:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "52:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 55f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 54f\n"
       "53:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4f60f0c8  // bfdot v8.4s, v6.8h, v0.h[1]\n"
       ".inst 0x4f61f0cc  // bfdot v12.4s, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f60f0e9  // bfdot v9.4s, v7.8h, v0.h[1]\n"
       ".inst 0x4f61f0ed  // bfdot v13.4s, v7.8h, v1.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f60f0ca  // bfdot v10.4s, v6.8h, v0.h[1]\n"
       ".inst 0x4f61f0ce  // bfdot v14.4s, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x80]\n"
@@ -588,32 +588,32 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f61f8ce  // bfdot v14.4s, v6.8h, v1.h[3]\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 53b\n"
       "54:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f60f0c8  // bfdot v8.4s, v6.8h, v0.h[1]\n"
       ".inst 0x4f61f0cc  // bfdot v12.4s, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f60f0e9  // bfdot v9.4s, v7.8h, v0.h[1]\n"
       ".inst 0x4f61f0ed  // bfdot v13.4s, v7.8h, v1.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
@@ -647,283 +647,283 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
       "55:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 59f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 59f\n"
+      "cmp x27, #0x2\n"
       "blt 57f\n"
       "56:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x2\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x2\n"
+      "cmp x27, #0x2\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       "bge 56b\n"
-      "cbz x26, 59f\n"
       "57:"  // Height 2: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
-      "ldr h1, [x24, #0x0]\n"
+      "cbz x27, 59f\n"
+      "ldr h0, [x26, #0x0]\n"
+      "ldr h1, [x25, #0x0]\n"
       "58:"  // Height 2: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       "59:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 50b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 60f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
       "60:"  // Height 2: No activation
       "cmp x11, #0x10\n"
       "bge 69f\n"
       "tbz x11, #3, 64f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 62f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 61f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
       "b 68f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x11, #0, 68f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
       "b 68f\n"
       "62:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x11, #1, 63f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
       "b 68f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x11, #0, 68f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
       "b 68f\n"
       "64:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x11, #2, 66f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 65f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
       "b 68f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x11, #0, 68f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
       "b 68f\n"
       "66:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x11, #1, 67f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
       "b 68f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
       "68:"  // Height 2: Partial direct writeback: Done
       "b 70f\n"
       "69:"  // Height 2: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
       "70:"  // Height 2: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 37b\n"
       "b 212f\n"
       "71:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "72:"  // Height 3: Column loop
-      "cbz x9, 73f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 73f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
       "mov v13.16b, v9.16b\n"
-      "add x9, x9, #0x40\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 84f\n"
       "73:"  // Height 3: no bias
       "tbz %x[flags], #0, 83f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 82f\n"
       "tbz x11, #3, 77f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 75f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 74f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 81f\n"
       "74:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 81f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 81f\n"
       "75:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x11, #1, 76f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 81f\n"
       "76:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 81f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 81f\n"
       "77:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x11, #2, 79f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 78f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 81f\n"
       "78:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 81f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
       "b 81f\n"
       "79:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x11, #1, 80f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
       "b 81f\n"
       "80:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
       "81:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 84f\n"
       "82:"  // Height 3: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
       "b 84f\n"
       "83:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -939,62 +939,62 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "84:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "85:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 86f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 87f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 87f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 87f\n"
       "86:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "87:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 90f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 89f\n"
       "88:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x8\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "cmp x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f60f0c8  // bfdot v8.4s, v6.8h, v0.h[1]\n"
       ".inst 0x4f61f0cc  // bfdot v12.4s, v6.8h, v1.h[1]\n"
       ".inst 0x4f62f0d0  // bfdot v16.4s, v6.8h, v2.h[1]\n"
@@ -1041,34 +1041,34 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f62f8d2  // bfdot v18.4s, v6.8h, v2.h[3]\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 88b\n"
       "89:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "ldr q7, [x10, #0x30]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -1120,19 +1120,19 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
       "90:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 94f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 94f\n"
+      "cmp x27, #0x2\n"
       "blt 92f\n"
       "91:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x2\n"
-      "ldr s2, [x23], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x2\n"
+      "cmp x27, #0x2\n"
+      "ldr s2, [x24], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
@@ -1147,15 +1147,15 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       "bge 91b\n"
-      "cbz x26, 94f\n"
       "92:"  // Height 3: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
-      "ldr h1, [x24, #0x0]\n"
-      "ldr h2, [x23, #0x0]\n"
+      "cbz x27, 94f\n"
+      "ldr h0, [x26, #0x0]\n"
+      "ldr h1, [x25, #0x0]\n"
+      "ldr h2, [x24, #0x0]\n"
       "93:"  // Height 3: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
@@ -1171,297 +1171,297 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       "94:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 85b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 95f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "95:"  // Height 3: No activation
       "cmp x11, #0x10\n"
       "bge 104f\n"
       "tbz x11, #3, 99f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 97f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 96f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 103f\n"
       "96:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x11, #0, 103f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 103f\n"
       "97:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x11, #1, 98f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 103f\n"
       "98:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x11, #0, 103f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 103f\n"
       "99:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x11, #2, 101f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 100f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 103f\n"
       "100:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x11, #0, 103f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 103f\n"
       "101:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x11, #1, 102f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 103f\n"
       "102:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "103:"  // Height 3: Partial direct writeback: Done
       "b 105f\n"
       "104:"  // Height 3: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "105:"  // Height 3: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 72b\n"
       "b 212f\n"
       "106:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "107:"  // Height 4: Column loop
-      "cbz x9, 108f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 108f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
       "b 119f\n"
       "108:"  // Height 4: no bias
       "tbz %x[flags], #0, 118f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 117f\n"
       "tbz x11, #3, 112f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 110f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 109f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 116f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 116f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 116f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x11, #1, 111f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 116f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 116f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 116f\n"
       "112:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x11, #2, 114f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 113f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 116f\n"
       "113:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 116f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 116f\n"
       "114:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x11, #1, 115f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 116f\n"
       "115:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "116:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 119f\n"
       "117:"  // Height 4: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "b 119f\n"
       "118:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1481,69 +1481,69 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "119:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "120:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 122f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 122f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 122f\n"
       "121:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "122:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 125f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 124f\n"
       "123:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       ".inst 0x4f43f0f7  // bfdot v23.4s, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -1604,40 +1604,40 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f63f8d6  // bfdot v22.4s, v6.8h, v3.h[3]\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4f63f8f7  // bfdot v23.4s, v7.8h, v3.h[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 123b\n"
       "124:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ef  // bfdot v15.4s, v7.8h, v1.h[0]\n"
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
@@ -1703,19 +1703,19 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
       ".inst 0x4f63f8f7  // bfdot v23.4s, v7.8h, v3.h[3]\n"
       "125:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 129f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 129f\n"
+      "cmp x27, #0x2\n"
       "blt 127f\n"
       "126:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x2\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x2\n"
+      "cmp x27, #0x2\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
@@ -1735,16 +1735,16 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       ".inst 0x4f43f0f7  // bfdot v23.4s, v7.8h, v3.h[0]\n"
       "bge 126b\n"
-      "cbz x26, 129f\n"
       "127:"  // Height 4: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
-      "ldr h1, [x24, #0x0]\n"
-      "ldr h2, [x23, #0x0]\n"
-      "ldr h3, [x22, #0x0]\n"
+      "cbz x27, 129f\n"
+      "ldr h0, [x26, #0x0]\n"
+      "ldr h1, [x25, #0x0]\n"
+      "ldr h2, [x24, #0x0]\n"
+      "ldr h3, [x23, #0x0]\n"
       "128:"  // Height 4: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
@@ -1764,352 +1764,352 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f42f0f3  // bfdot v19.4s, v7.8h, v2.h[0]\n"
       ".inst 0x4f43f0f7  // bfdot v23.4s, v7.8h, v3.h[0]\n"
       "129:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 120b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 130f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
       "130:"  // Height 4: No activation
       "cmp x11, #0x10\n"
       "bge 139f\n"
       "tbz x11, #3, 134f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 132f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 131f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
       "b 138f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x11, #0, 138f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
       "b 138f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x11, #1, 133f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
       "b 138f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x11, #0, 138f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
       "b 138f\n"
       "134:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x11, #2, 136f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 135f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
       "b 138f\n"
       "135:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x11, #0, 138f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
       "b 138f\n"
       "136:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x11, #1, 137f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
       "b 138f\n"
       "137:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
       "138:"  // Height 4: Partial direct writeback: Done
       "b 140f\n"
       "139:"  // Height 4: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
       "140:"  // Height 4: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 107b\n"
       "b 212f\n"
       "141:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "142:"  // Height 5: Column loop
-      "cbz x9, 143f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 143f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
       "b 154f\n"
       "143:"  // Height 5: no bias
       "tbz %x[flags], #0, 153f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 152f\n"
       "tbz x11, #3, 147f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 145f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 144f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 151f\n"
       "144:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 151f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 151f\n"
       "145:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x11, #1, 146f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 151f\n"
       "146:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 151f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 151f\n"
       "147:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x11, #2, 149f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 148f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 151f\n"
       "148:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 151f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 151f\n"
       "149:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x11, #1, 150f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 151f\n"
       "150:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "151:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 154f\n"
       "152:"  // Height 5: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
       "b 154f\n"
       "153:"  // Height 5: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2133,74 +2133,74 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "154:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "155:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 156f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 157f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 157f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 157f\n"
       "156:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "157:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 160f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 159f\n"
       "158:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f44f0d8  // bfdot v24.4s, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x8\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
       ".inst 0x4f44f0f9  // bfdot v25.4s, v7.8h, v4.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f44f0da  // bfdot v26.4s, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
@@ -2277,45 +2277,45 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f64f8da  // bfdot v26.4s, v6.8h, v4.h[3]\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4f63f8f7  // bfdot v23.4s, v7.8h, v3.h[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4f64f8fb  // bfdot v27.4s, v7.8h, v4.h[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 158b\n"
       "159:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f44f0d8  // bfdot v24.4s, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x8\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
       ".inst 0x4f44f0f9  // bfdot v25.4s, v7.8h, v4.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f44f0da  // bfdot v26.4s, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f40f0eb  // bfdot v11.4s, v7.8h, v0.h[0]\n"
@@ -2396,21 +2396,21 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f63f8f7  // bfdot v23.4s, v7.8h, v3.h[3]\n"
       ".inst 0x4f64f8fb  // bfdot v27.4s, v7.8h, v4.h[3]\n"
       "160:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 164f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 164f\n"
+      "cmp x27, #0x2\n"
       "blt 162f\n"
       "161:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x2\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x2\n"
+      "cmp x27, #0x2\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
       ".inst 0x4f44f0d8  // bfdot v24.4s, v6.8h, v4.h[0]\n"
@@ -2433,17 +2433,17 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f43f0f7  // bfdot v23.4s, v7.8h, v3.h[0]\n"
       ".inst 0x4f44f0fb  // bfdot v27.4s, v7.8h, v4.h[0]\n"
       "bge 161b\n"
-      "cbz x26, 164f\n"
       "162:"  // Height 5: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
-      "ldr h1, [x24, #0x0]\n"
-      "ldr h2, [x23, #0x0]\n"
-      "ldr h3, [x22, #0x0]\n"
-      "ldr h4, [x21, #0x0]\n"
+      "cbz x27, 164f\n"
+      "ldr h0, [x26, #0x0]\n"
+      "ldr h1, [x25, #0x0]\n"
+      "ldr h2, [x24, #0x0]\n"
+      "ldr h3, [x23, #0x0]\n"
+      "ldr h4, [x22, #0x0]\n"
       "163:"  // Height 5: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
@@ -2467,410 +2467,410 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f43f0f7  // bfdot v23.4s, v7.8h, v3.h[0]\n"
       ".inst 0x4f44f0fb  // bfdot v27.4s, v7.8h, v4.h[0]\n"
       "164:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 155b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 165f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
       "165:"  // Height 5: No activation
       "cmp x11, #0x10\n"
       "bge 174f\n"
       "tbz x11, #3, 169f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 167f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 166f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 173f\n"
       "166:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x11, #0, 173f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 173f\n"
       "167:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x11, #1, 168f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 173f\n"
       "168:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x11, #0, 173f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 173f\n"
       "169:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x11, #2, 171f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 170f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 173f\n"
       "170:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x11, #0, 173f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 173f\n"
       "171:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x11, #1, 172f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 173f\n"
       "172:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "173:"  // Height 5: Partial direct writeback: Done
       "b 175f\n"
       "174:"  // Height 5: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "175:"  // Height 5: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 142b\n"
       "b 212f\n"
       "176:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "177:"  // Height 6: Column loop
-      "cbz x9, 178f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 178f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
-      "mov v28.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
+      "mov v28.16b, v8.16b\n"
       "mov v29.16b, v9.16b\n"
       "mov v30.16b, v10.16b\n"
       "mov v31.16b, v11.16b\n"
       "b 189f\n"
       "178:"  // Height 6: no bias
       "tbz %x[flags], #0, 188f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 187f\n"
       "tbz x11, #3, 182f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 180f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 179f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 186f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 186f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 186f\n"
       "180:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x11, #1, 181f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 186f\n"
       "181:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 186f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 186f\n"
       "182:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x11, #2, 184f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 183f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 186f\n"
       "183:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 186f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 186f\n"
       "184:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x11, #1, 185f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 186f\n"
       "185:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "186:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 189f\n"
       "187:"  // Height 6: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "b 189f\n"
       "188:"  // Height 6: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2898,82 +2898,82 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "189:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "190:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 191f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 192f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 192f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 192f\n"
       "191:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "192:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 195f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 194f\n"
       "193:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f44f0d8  // bfdot v24.4s, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f45f0dc  // bfdot v28.4s, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x10\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
+      "cmp x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f44f0f9  // bfdot v25.4s, v7.8h, v4.h[0]\n"
       ".inst 0x4f45f0fd  // bfdot v29.4s, v7.8h, v5.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f44f0da  // bfdot v26.4s, v6.8h, v4.h[0]\n"
       ".inst 0x4f45f0de  // bfdot v30.4s, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -3063,51 +3063,51 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f65f8de  // bfdot v30.4s, v6.8h, v5.h[3]\n"
       "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f60f8eb  // bfdot v11.4s, v7.8h, v0.h[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4f61f8ef  // bfdot v15.4s, v7.8h, v1.h[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4f62f8f3  // bfdot v19.4s, v7.8h, v2.h[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4f63f8f7  // bfdot v23.4s, v7.8h, v3.h[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4f64f8fb  // bfdot v27.4s, v7.8h, v4.h[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x4f65f8ff  // bfdot v31.4s, v7.8h, v5.h[3]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 193b\n"
       "194:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f44f0d8  // bfdot v24.4s, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f45f0dc  // bfdot v28.4s, v6.8h, v5.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x22, x22, #0x10\n"
+      ".inst 0x4f40f0e9  // bfdot v9.4s, v7.8h, v0.h[0]\n"
       ".inst 0x4f41f0ed  // bfdot v13.4s, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4f42f0f1  // bfdot v17.4s, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4f43f0f5  // bfdot v21.4s, v7.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f44f0f9  // bfdot v25.4s, v7.8h, v4.h[0]\n"
       ".inst 0x4f45f0fd  // bfdot v29.4s, v7.8h, v5.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0ce  // bfdot v14.4s, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f42f0d2  // bfdot v18.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f44f0da  // bfdot v26.4s, v6.8h, v4.h[0]\n"
       ".inst 0x4f45f0de  // bfdot v30.4s, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -3202,21 +3202,21 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f64f8fb  // bfdot v27.4s, v7.8h, v4.h[3]\n"
       ".inst 0x4f65f8ff  // bfdot v31.4s, v7.8h, v5.h[3]\n"
       "195:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 199f\n"
-      "cmp x26, #0x2\n"
+      "cbz x27, 199f\n"
+      "cmp x27, #0x2\n"
       "blt 197f\n"
       "196:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x2\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x2\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x2\n"
+      "cmp x27, #0x2\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
@@ -3244,18 +3244,18 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f44f0fb  // bfdot v27.4s, v7.8h, v4.h[0]\n"
       ".inst 0x4f45f0ff  // bfdot v31.4s, v7.8h, v5.h[0]\n"
       "bge 196b\n"
-      "cbz x26, 199f\n"
       "197:"  // Height 6: Multiply loop: Skip odd blocks
-      "ldr h0, [x25, #0x0]\n"
-      "ldr h1, [x24, #0x0]\n"
-      "ldr h2, [x23, #0x0]\n"
-      "ldr h3, [x22, #0x0]\n"
-      "ldr h4, [x21, #0x0]\n"
-      "ldr h5, [x20, #0x0]\n"
+      "cbz x27, 199f\n"
+      "ldr h0, [x26, #0x0]\n"
+      "ldr h1, [x25, #0x0]\n"
+      "ldr h2, [x24, #0x0]\n"
+      "ldr h3, [x23, #0x0]\n"
+      "ldr h4, [x22, #0x0]\n"
+      "ldr h5, [x21, #0x0]\n"
       "198:"  // Height 6: Multiply loop: Ragged operand read: Done
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x4f40f0c8  // bfdot v8.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f41f0cc  // bfdot v12.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f42f0d0  // bfdot v16.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f43f0d4  // bfdot v20.4s, v6.8h, v3.h[0]\n"
@@ -3283,249 +3283,249 @@ void a64_hybrid_bf16fp32_dot_6x16 (
       ".inst 0x4f44f0fb  // bfdot v27.4s, v7.8h, v4.h[0]\n"
       ".inst 0x4f45f0ff  // bfdot v31.4s, v7.8h, v5.h[0]\n"
       "199:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 190b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbz %x[flags], #1, 200f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v31.4s, v31.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v31.4s, v31.4s, v0.4s\n"
       "200:"  // Height 6: No activation
       "cmp x11, #0x10\n"
       "bge 209f\n"
       "tbz x11, #3, 204f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 202f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v30.4s }, [x20], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 201f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
-      "st1 { v31.s }[2], [x20]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
+      "st1 { v31.s }[2], [x21]\n"
       "b 208f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x11, #0, 208f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
-      "str s31, [x20, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
+      "str s31, [x21, #0x0]\n"
       "b 208f\n"
       "202:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x11, #1, 203f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
       "b 208f\n"
       "203:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x11, #0, 208f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
       "b 208f\n"
       "204:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x11, #2, 206f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 205f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
       "b 208f\n"
       "205:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x11, #0, 208f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
       "b 208f\n"
       "206:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x11, #1, 207f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
       "b 208f\n"
       "207:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
       "208:"  // Height 6: Partial direct writeback: Done
       "b 210f\n"
       "209:"  // Height 6: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
-      "str q28, [x20, #0x0]\n"
-      "str q29, [x20, #0x10]\n"
-      "str q30, [x20, #0x20]\n"
-      "str q31, [x20, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
+      "str q28, [x21, #0x0]\n"
+      "str q29, [x21, #0x10]\n"
+      "str q30, [x21, #0x20]\n"
+      "str q31, [x21, #0x30]\n"
       "210:"  // Height 6: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 177b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 212f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 211f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "211:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "212:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp
index 0fa358e848..5a000c69af 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -103,23 +103,23 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 75f\n"
       "beq 38f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x9, 3f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 3f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
-      "add x9, x9, #0x40\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 15f\n"
@@ -128,65 +128,65 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "cmp x11, #0x10\n"
       "bge 12f\n"
       "tbz x11, #3, 7f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #2, 5f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
       "tbz x11, #1, 4f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v16.s }[2], [x28]\n"
+      "ld1 { v16.s }[2], [x9]\n"
       "b 11f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 11f\n"
-      "ldr s16, [x28, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
       "b 11f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x11, #1, 6f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 11f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 11f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 11f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x11, #2, 9f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #1, 8f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 11f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 11f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 11f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x11, #1, 10f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 11f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x9, #0x0]\n"
+      "mov x20, #0x0\n"
       "11:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 13f\n"
       "12:"  // Height 1: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
       "13:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -207,37 +207,31 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "15:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 18f\n"
       "17:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "18:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 21f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 20f\n"
       "19:"  // Height 1: Multiply loop: Main loop head
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
@@ -250,6 +244,7 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
@@ -266,20 +261,19 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       "ldr q6, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 19b\n"
       "20:"  // Height 1: Multiply loop: Single iteration only
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x10, #0x0]\n"
-      "sub x26, x26, #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
@@ -292,6 +286,7 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
@@ -308,22 +303,22 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       "ldr q6, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "21:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 26f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 26f\n"
+      "cmp x27, #0x4\n"
       "blt 23f\n"
       "22:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "ldr d1, [x26], #0x8\n"
       "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x4\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      "trn1 v0.2d, v1.2d, v2.2d\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
@@ -335,21 +330,22 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x80\n"
       "bge 22b\n"
-      "cbz x26, 26f\n"
       "23:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 24f\n"
-      "ldr s1, [x25], #0x4\n"
-      "tbz x26, #0, 25f\n"
-      "ld1 { v1.h }[2], [x25]\n"
+      "cbz x27, 26f\n"
+      "tbz x27, #1, 24f\n"
+      "ldr s1, [x26], #0x4\n"
+      "tbz x27, #0, 25f\n"
+      "ld1 { v1.h }[2], [x26]\n"
       "b 25f\n"
       "24:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
       "25:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
       "ldr q7, [x10, #0x0]\n"
       "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
@@ -365,206 +361,198 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       "26:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "27:"  // Height 1: No activation
       "uzp1 v8.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
+      "tbz %x[flags], #1, 27f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "27:"  // Height 1: No activation
+      "cmp x11, #0x10\n"
       "bge 36f\n"
       "tbz x11, #3, 31f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 29f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 28f\n"
-      "str d11, [x28], #0x8\n"
+      "str d11, [x9], #0x8\n"
       "tbz x11, #0, 35f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 35f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x11, #0, 35f\n"
-      "str s11, [x28, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
       "b 35f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x11, #1, 30f\n"
-      "str d10, [x28], #0x8\n"
+      "str d10, [x9], #0x8\n"
       "tbz x11, #0, 35f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 35f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x11, #0, 35f\n"
-      "str s10, [x28, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
       "b 35f\n"
       "31:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x11, #2, 33f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 32f\n"
-      "str d9, [x28], #0x8\n"
+      "str d9, [x9], #0x8\n"
       "tbz x11, #0, 35f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 35f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x11, #0, 35f\n"
-      "str s9, [x28, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
       "b 35f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x11, #1, 34f\n"
-      "str d8, [x28], #0x8\n"
+      "str d8, [x9], #0x8\n"
       "tbz x11, #0, 35f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 35f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "35:"  // Height 1: Partial direct writeback: Done
       "b 37f\n"
       "36:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "37:"  // Height 1: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 224f\n"
       "38:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "39:"  // Height 2: Column loop
-      "cbz x9, 40f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 40f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
-      "add x9, x9, #0x40\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 52f\n"
       "40:"  // Height 2: no bias
       "tbz %x[flags], #0, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "bge 49f\n"
       "tbz x11, #3, 44f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 42f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 41f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 48f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 48f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 48f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 48f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x11, #1, 43f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 48f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 48f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 48f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 48f\n"
       "44:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x11, #2, 46f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 45f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 48f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 48f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 48f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 48f\n"
       "46:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x11, #1, 47f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 48f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 48f\n"
       "47:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "48:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 50f\n"
       "49:"  // Height 2: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "50:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -585,57 +573,50 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "52:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "53:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 54f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 55f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 55f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 55f\n"
       "54:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "55:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 58f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 57f\n"
       "56:"  // Height 2: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "sub x26, x26, #0x8\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
-      "ldr q2, [x24, #0x0]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
@@ -650,33 +631,35 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       "ldr q6, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "bge 56b\n"
       "57:"  // Height 2: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "sub x26, x26, #0x8\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
@@ -693,53 +676,58 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       "ldr q6, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      "sub x27, x27, #0x8\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "58:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 63f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 63f\n"
+      "cmp x27, #0x4\n"
       "blt 60f\n"
       "59:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
+      "sub x27, x27, #0x4\n"
       "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x4\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q7, [x10, #0x10]\n"
-      "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x10, #0x20]\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x10, #0x40]\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x10, #0x60]\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x80\n"
       "bge 59b\n"
-      "cbz x26, 63f\n"
       "60:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 61f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "tbz x26, #0, 62f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
+      "cbz x27, 63f\n"
+      "tbz x27, #1, 61f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "tbz x27, #0, 62f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
       "b 62f\n"
       "61:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
       "62:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "trn1 v0.2d, v1.2d, v2.2d\n"
       "ldr q7, [x10, #0x0]\n"
       "ldr q6, [x10, #0x10]\n"
+      "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
@@ -752,148 +740,148 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       "63:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 53b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
-      "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "64:"  // Height 2: No activation
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
+      "tbz %x[flags], #1, 64f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v7.4s, v7.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v7.4s, v7.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "64:"  // Height 2: No activation
+      "cmp x11, #0x10\n"
       "bge 73f\n"
       "tbz x11, #3, 68f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
       "tbz x11, #2, 66f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
       "tbz x11, #1, 65f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
       "tbz x11, #0, 72f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
       "b 72f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x11, #0, 72f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
       "b 72f\n"
       "66:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x11, #1, 67f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
       "tbz x11, #0, 72f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
       "b 72f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x11, #0, 72f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
       "b 72f\n"
       "68:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x11, #2, 70f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
       "tbz x11, #1, 69f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
       "tbz x11, #0, 72f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
       "b 72f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x11, #0, 72f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
       "b 72f\n"
       "70:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x11, #1, 71f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
       "tbz x11, #0, 72f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
       "b 72f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
       "72:"  // Height 2: Partial direct writeback: Done
       "b 74f\n"
       "73:"  // Height 2: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
       "74:"  // Height 2: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 39b\n"
       "b 224f\n"
       "75:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "76:"  // Height 3: Column loop
-      "cbz x9, 77f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 77f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
@@ -903,111 +891,111 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "b 89f\n"
       "77:"  // Height 3: no bias
       "tbz %x[flags], #0, 88f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 86f\n"
       "tbz x11, #3, 81f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #2, 79f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
       "tbz x11, #1, 78f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
       "tbz x11, #0, 85f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
       "b 85f\n"
       "78:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 85f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
       "b 85f\n"
       "79:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x11, #1, 80f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
       "tbz x11, #0, 85f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 85f\n"
       "80:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 85f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 85f\n"
       "81:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x11, #2, 83f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #1, 82f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
       "tbz x11, #0, 85f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 85f\n"
       "82:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 85f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 85f\n"
       "83:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x11, #1, 84f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
       "tbz x11, #0, 85f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 85f\n"
       "84:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
       "85:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 87f\n"
       "86:"  // Height 3: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
       "87:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1044,85 +1032,85 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "89:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "90:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 91f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 92f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 92f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 92f\n"
       "91:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "92:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 95f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 94f\n"
       "93:"  // Height 3: Multiply loop: Main loop head
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
@@ -1134,55 +1122,53 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "add x10, x10, #0x100\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 93b\n"
       "94:"  // Height 3: Multiply loop: Single iteration only
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
@@ -1203,30 +1189,29 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       "95:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 100f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 100f\n"
+      "cmp x27, #0x4\n"
       "blt 97f\n"
       "96:"  // Height 3: Multiply loop: Odd block loop
-      "movi v4.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x4\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
+      "ldr d3, [x24], #0x8\n"
       "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "sub x27, x27, #0x4\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
@@ -1235,32 +1220,31 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "bge 96b\n"
-      "cbz x26, 100f\n"
       "97:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 98f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "tbz x26, #0, 99f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
+      "cbz x27, 100f\n"
+      "tbz x27, #1, 98f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "tbz x27, #0, 99f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
       "b 99f\n"
       "98:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
       "99:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v4.16b, #0x0\n"
       "ldr q7, [x10, #0x0]\n"
-      "trn1 v0.2d, v1.2d, v2.2d\n"
       "ldr q6, [x10, #0x10]\n"
+      "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
@@ -1286,60 +1270,21 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "100:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 90b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "tbz %x[flags], #1, 101f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "101:"  // Height 3: No activation
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
@@ -1347,125 +1292,156 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "uzp1 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
+      "tbz %x[flags], #1, 101f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v7.4s, v7.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v7.4s, v7.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "101:"  // Height 3: No activation
+      "cmp x11, #0x10\n"
       "bge 110f\n"
       "tbz x11, #3, 105f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 103f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 102f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
       "tbz x11, #0, 109f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 109f\n"
       "102:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x11, #0, 109f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 109f\n"
       "103:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x11, #1, 104f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
       "tbz x11, #0, 109f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 109f\n"
       "104:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x11, #0, 109f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 109f\n"
       "105:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x11, #2, 107f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 106f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
       "tbz x11, #0, 109f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 109f\n"
       "106:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x11, #0, 109f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 109f\n"
       "107:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x11, #1, 108f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
       "tbz x11, #0, 109f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 109f\n"
       "108:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "109:"  // Height 3: Partial direct writeback: Done
       "b 111f\n"
       "110:"  // Height 3: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "111:"  // Height 3: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 76b\n"
       "b 224f\n"
       "112:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "113:"  // Height 4: Column loop
-      "cbz x9, 114f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 114f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
@@ -1475,132 +1451,132 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "b 126f\n"
       "114:"  // Height 4: no bias
       "tbz %x[flags], #0, 125f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 123f\n"
       "tbz x11, #3, 118f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 116f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 115f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
       "tbz x11, #0, 122f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 122f\n"
       "115:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 122f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 122f\n"
       "116:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x11, #1, 117f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
       "tbz x11, #0, 122f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 122f\n"
       "117:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 122f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 122f\n"
       "118:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x11, #2, 120f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 119f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
       "tbz x11, #0, 122f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 122f\n"
       "119:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 122f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 122f\n"
       "120:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x11, #1, 121f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
       "tbz x11, #0, 122f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 122f\n"
       "121:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "122:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 124f\n"
       "123:"  // Height 4: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "124:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1637,155 +1613,155 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "126:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "127:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 128f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 129f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 129f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 129f\n"
       "128:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "129:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 132f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 131f\n"
       "130:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "add x23, x23, #0x10\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec75  // bfmmla v21.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e47ec2a  // bfmmla v10.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec72  // bfmmla v18.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec76  // bfmmla v22.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xf0]\n"
       "add x10, x10, #0x100\n"
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 130b\n"
       "131:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x8\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x10, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e46ec2c  // bfmmla v12.4s, v1.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec74  // bfmmla v20.4s, v3.8h, v6.8h\n"
       "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e47ec29  // bfmmla v9.4s, v1.8h, v7.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec71  // bfmmla v17.4s, v3.8h, v7.8h\n"
       "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e46ec2d  // bfmmla v13.4s, v1.8h, v6.8h\n"
@@ -1803,18 +1779,18 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       "132:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 137f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 137f\n"
+      "cmp x27, #0x4\n"
       "blt 134f\n"
       "133:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x4\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
+      "cmp x27, #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
@@ -1841,29 +1817,29 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "bge 133b\n"
-      "cbz x26, 137f\n"
       "134:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 135f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "tbz x26, #0, 136f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
+      "cbz x27, 137f\n"
+      "tbz x27, #1, 135f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "tbz x27, #0, 136f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
       "b 136f\n"
       "135:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
       "136:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "trn1 v0.2d, v1.2d, v2.2d\n"
       "ldr q7, [x10, #0x0]\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
       "ldr q6, [x10, #0x10]\n"
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
@@ -1888,64 +1864,25 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "137:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 127b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "tbz %x[flags], #1, 138f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "138:"  // Height 4: No activation
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
@@ -1955,152 +1892,191 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
+      "tbz %x[flags], #1, 138f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v7.4s, v7.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v7.4s, v7.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "138:"  // Height 4: No activation
+      "cmp x11, #0x10\n"
       "bge 147f\n"
       "tbz x11, #3, 142f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
       "tbz x11, #2, 140f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
       "tbz x11, #1, 139f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
       "tbz x11, #0, 146f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 146f\n"
       "139:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x11, #0, 146f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x11, #1, 141f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
       "tbz x11, #0, 146f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 146f\n"
       "141:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x11, #0, 146f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x11, #2, 144f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
       "tbz x11, #1, 143f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
       "tbz x11, #0, 146f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 146f\n"
       "143:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x11, #0, 146f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x11, #1, 145f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
       "tbz x11, #0, 146f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 146f\n"
       "145:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "146:"  // Height 4: Partial direct writeback: Done
       "b 148f\n"
       "147:"  // Height 4: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "148:"  // Height 4: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 113b\n"
       "b 224f\n"
       "149:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "150:"  // Height 5: Column loop
-      "cbz x9, 151f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 151f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
-      "mov v24.16b, v8.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v22.16b, v14.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v23.16b, v15.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v28.16b, v12.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v29.16b, v13.16b\n"
@@ -2111,153 +2087,153 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "b 163f\n"
       "151:"  // Height 5: no bias
       "tbz %x[flags], #0, 162f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 160f\n"
       "tbz x11, #3, 155f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #2, 153f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
       "tbz x11, #1, 152f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
       "tbz x11, #0, 159f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
       "b 159f\n"
       "152:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 159f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
       "b 159f\n"
       "153:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x11, #1, 154f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
       "tbz x11, #0, 159f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 159f\n"
       "154:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 159f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 159f\n"
       "155:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x11, #2, 157f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #1, 156f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
       "tbz x11, #0, 159f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 159f\n"
       "156:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 159f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 159f\n"
       "157:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x11, #1, 158f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x11, #0, 159f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 159f\n"
       "158:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "159:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 161f\n"
       "160:"  // Height 5: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
       "161:"  // Height 5: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2310,97 +2286,98 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "163:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "164:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 165f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 166f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 166f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 166f\n"
       "165:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "166:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 169f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 168f\n"
       "167:"  // Height 5: Multiply loop: Main loop head
-      "movi v6.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
       "ldr q6, [x10, #0x10]\n"
-      "cmp x26, #0x10\n"
-      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
@@ -2429,61 +2406,58 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
+      "ldr q5, [x22, #0x0]\n"
       "bge 167b\n"
       "168:"  // Height 5: Multiply loop: Single iteration only
-      "movi v6.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
       "ldr q6, [x10, #0x10]\n"
-      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x22, x22, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
@@ -2523,29 +2497,28 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
       "169:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 174f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 174f\n"
+      "cmp x27, #0x4\n"
       "blt 171f\n"
       "170:"  // Height 5: Multiply loop: Odd block loop
-      "movi v7.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x4\n"
-      "ldr d4, [x22], #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d5, [x22], #0x8\n"
       "ldr q6, [x10, #0x0]\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
@@ -2564,37 +2537,36 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9e  // bfmmla v30.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "bge 170b\n"
-      "cbz x26, 174f\n"
       "171:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 172f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "tbz x26, #0, 173f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
-      "ld1 { v5.h }[2], [x21]\n"
+      "cbz x27, 174f\n"
+      "tbz x27, #1, 172f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "tbz x27, #0, 173f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
+      "ld1 { v5.h }[2], [x22]\n"
       "b 173f\n"
       "172:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
-      "ldr h5, [x21, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
+      "ldr h5, [x22, #0x0]\n"
       "173:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "movi v6.16b, #0x0\n"
       "ldr q7, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
@@ -2632,84 +2604,29 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "174:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 164b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "tbz %x[flags], #1, 175f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
-      "175:"  // Height 5: No activation
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
@@ -2721,175 +2638,222 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "uzp1 v25.2d, v25.2d, v29.2d\n"
       "uzp1 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v27.2d, v27.2d, v31.2d\n"
+      "tbz %x[flags], #1, 175f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v7.4s, v7.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v7.4s, v7.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "175:"  // Height 5: No activation
+      "cmp x11, #0x10\n"
       "bge 184f\n"
       "tbz x11, #3, 179f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 177f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 176f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
       "tbz x11, #0, 183f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 183f\n"
       "176:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x11, #0, 183f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 183f\n"
       "177:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x11, #1, 178f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
       "tbz x11, #0, 183f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 183f\n"
       "178:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x11, #0, 183f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 183f\n"
       "179:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x11, #2, 181f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 180f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x11, #0, 183f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 183f\n"
       "180:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x11, #0, 183f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 183f\n"
       "181:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x11, #1, 182f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x11, #0, 183f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 183f\n"
       "182:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "183:"  // Height 5: Partial direct writeback: Done
       "b 185f\n"
       "184:"  // Height 5: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "185:"  // Height 5: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 150b\n"
       "b 224f\n"
       "186:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "187:"  // Height 6: Column loop
-      "cbz x9, 188f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 188f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
-      "mov v24.16b, v8.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v22.16b, v14.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v23.16b, v15.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v28.16b, v12.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v29.16b, v13.16b\n"
@@ -2900,174 +2864,174 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "b 200f\n"
       "188:"  // Height 6: no bias
       "tbz %x[flags], #0, 199f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 197f\n"
       "tbz x11, #3, 192f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 190f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 189f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 196f\n"
       "189:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 196f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 196f\n"
       "190:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x11, #1, 191f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 196f\n"
       "191:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 196f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 196f\n"
       "192:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x11, #2, 194f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 193f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 196f\n"
       "193:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 196f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 196f\n"
       "194:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x11, #1, 195f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 196f\n"
       "195:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "196:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 198f\n"
       "197:"  // Height 6: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "198:"  // Height 6: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -3120,103 +3084,104 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "200:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "201:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 202f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 203f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 203f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 203f\n"
       "202:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "203:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 206f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 205f\n"
       "204:"  // Height 6: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
+      "sub x27, x27, #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
       "ldr q6, [x10, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x8\n"
-      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "cmp x26, #0x10\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "cmp x27, #0x10\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e47ec28  // bfmmla v8.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec70  // bfmmla v16.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecb8  // bfmmla v24.4s, v5.8h, v7.8h\n"
@@ -3245,65 +3210,64 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec2b  // bfmmla v11.4s, v1.8h, v7.8h\n"
       ".inst 0x6e47ec73  // bfmmla v19.4s, v3.8h, v7.8h\n"
       ".inst 0x6e47ecbb  // bfmmla v27.4s, v5.8h, v7.8h\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 204b\n"
       "205:"  // Height 6: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x8\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
+      "add x26, x26, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
+      "add x25, x25, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x10, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
       "ldr q6, [x10, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e46ec0c  // bfmmla v12.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e46ec9c  // bfmmla v28.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e47ec09  // bfmmla v9.4s, v0.8h, v7.8h\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e47ec51  // bfmmla v17.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e47ec99  // bfmmla v25.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e46ec9d  // bfmmla v29.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e47ec0a  // bfmmla v10.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e47ec52  // bfmmla v18.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9a  // bfmmla v26.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec0e  // bfmmla v14.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
@@ -3340,20 +3304,20 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec77  // bfmmla v23.4s, v3.8h, v6.8h\n"
       ".inst 0x6e46ecbf  // bfmmla v31.4s, v5.8h, v6.8h\n"
       "206:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 211f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 211f\n"
+      "cmp x27, #0x4\n"
       "blt 208f\n"
       "207:"  // Height 6: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x4\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x4\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x4\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "cmp x27, #0x4\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
       "ldr q6, [x10, #0x0]\n"
       "ldr q7, [x10, #0x10]\n"
@@ -3389,37 +3353,37 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "bge 207b\n"
-      "cbz x26, 211f\n"
       "208:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 209f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "ldr s6, [x20], #0x4\n"
-      "tbz x26, #0, 210f\n"
-      "ld1 { v1.h }[2], [x25]\n"
-      "ld1 { v2.h }[2], [x24]\n"
-      "ld1 { v3.h }[2], [x23]\n"
-      "ld1 { v4.h }[2], [x22]\n"
-      "ld1 { v5.h }[2], [x21]\n"
-      "ld1 { v6.h }[2], [x20]\n"
+      "cbz x27, 211f\n"
+      "tbz x27, #1, 209f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "ldr s6, [x21], #0x4\n"
+      "tbz x27, #0, 210f\n"
+      "ld1 { v1.h }[2], [x26]\n"
+      "ld1 { v2.h }[2], [x25]\n"
+      "ld1 { v3.h }[2], [x24]\n"
+      "ld1 { v4.h }[2], [x23]\n"
+      "ld1 { v5.h }[2], [x22]\n"
+      "ld1 { v6.h }[2], [x21]\n"
       "b 210f\n"
       "209:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr h1, [x25, #0x0]\n"
-      "ldr h2, [x24, #0x0]\n"
-      "ldr h3, [x23, #0x0]\n"
-      "ldr h4, [x22, #0x0]\n"
-      "ldr h5, [x21, #0x0]\n"
-      "ldr h6, [x20, #0x0]\n"
+      "ldr h1, [x26, #0x0]\n"
+      "ldr h2, [x25, #0x0]\n"
+      "ldr h3, [x24, #0x0]\n"
+      "ldr h4, [x23, #0x0]\n"
+      "ldr h5, [x22, #0x0]\n"
+      "ldr h6, [x21, #0x0]\n"
       "210:"  // Height 6: Multiply loop: Ragged operand read: Done
-      "trn1 v0.2d, v1.2d, v2.2d\n"
       "ldr q7, [x10, #0x0]\n"
+      "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
+      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
       "ldr q6, [x10, #0x10]\n"
-      ".inst 0x6e47ec08  // bfmmla v8.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec50  // bfmmla v16.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec98  // bfmmla v24.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x20]\n"
@@ -3443,96 +3407,41 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9e  // bfmmla v30.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e47ec53  // bfmmla v19.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9b  // bfmmla v27.4s, v4.8h, v7.8h\n"
       ".inst 0x6e46ec0f  // bfmmla v15.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9f  // bfmmla v31.4s, v4.8h, v6.8h\n"
       "211:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 201b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "tbz %x[flags], #1, 212f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
-      "212:"  // Height 6: No activation
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
@@ -3546,178 +3455,233 @@ void a64_hybrid_bf16fp32_mmla_6x16 (
       "uzp2 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v30.2d, v27.2d, v31.2d\n"
       "uzp2 v27.2d, v27.2d, v31.2d\n"
+      "tbz %x[flags], #1, 212f\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v7.4s, v7.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v7.4s, v7.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "212:"  // Height 6: No activation
+      "cmp x11, #0x10\n"
       "bge 221f\n"
       "tbz x11, #3, 216f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
       "tbz x11, #2, 214f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v29.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v29.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
       "tbz x11, #1, 213f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
       "tbz x11, #0, 220f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 220f\n"
       "213:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x11, #0, 220f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 220f\n"
       "214:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x11, #1, 215f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d29, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d29, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
       "tbz x11, #0, 220f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v29.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 220f\n"
       "215:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x11, #0, 220f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s29, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s29, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 220f\n"
       "216:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x11, #2, 218f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
       "tbz x11, #1, 217f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d28, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d28, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
       "tbz x11, #0, 220f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v28.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v28.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 220f\n"
       "217:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x11, #0, 220f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s28, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s28, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 220f\n"
       "218:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x11, #1, 219f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x11, #0, 220f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 220f\n"
       "219:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "220:"  // Height 6: Partial direct writeback: Done
       "b 222f\n"
       "221:"  // Height 6: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q23, [x21, #0x0]\n"
-      "str q28, [x21, #0x10]\n"
-      "str q29, [x21, #0x20]\n"
-      "str q30, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q23, [x22, #0x0]\n"
+      "str q28, [x22, #0x10]\n"
+      "str q29, [x22, #0x20]\n"
+      "str q30, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "222:"  // Height 6: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 187b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 224f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 223f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "223:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "224:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp
index 9157d29eba..19636548a0 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
 
@@ -101,138 +101,138 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 99f\n"
       "beq 50f\n"
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x16, %x[bias]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x16, 3f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
-      "ldr q11, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
+      "cbz x7, 3f\n"
+      "ldr q8, [x7, #0x0]\n"
+      "ldr q9, [x7, #0x10]\n"
+      "ldr q10, [x7, #0x20]\n"
+      "ldr q11, [x7, #0x30]\n"
+      "add x7, x7, #0x40\n"
       "b 22f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 21f\n"
       "cmp x8, #0x20\n"
       "bge 20f\n"
       "tbz x8, #4, 11f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
       "tbz x8, #3, 7f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "tbz x8, #2, 5f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "tbz x8, #1, 4f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "b 19f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "b 19f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_24
       "tbz x8, #1, 6f\n"
-      "ldr s11, [x15], #0x4\n"
-      "mov x19, #0x34\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "b 19f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 19f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "b 19f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_16
       "tbz x8, #2, 9f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "tbz x8, #1, 8f\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "b 19f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "b 19f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_16
       "tbz x8, #1, 10f\n"
-      "ldr s10, [x15], #0x4\n"
-      "mov x19, #0x24\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "b 19f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 19f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "b 19f\n"
       "11:"  // Height 1: Partial accumulate: partial_8_0
       "tbz x8, #3, 15f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "tbz x8, #2, 13f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "tbz x8, #1, 12f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "b 19f\n"
       "12:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "b 19f\n"
       "13:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x8, #1, 14f\n"
-      "ldr s9, [x15], #0x4\n"
-      "mov x19, #0x14\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "b 19f\n"
       "14:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 19f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "b 19f\n"
       "15:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x8, #2, 17f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "tbz x8, #1, 16f\n"
-      "mov x19, #0xc\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "b 19f\n"
       "16:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "b 19f\n"
       "17:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x8, #1, 18f\n"
-      "ldr s8, [x15], #0x4\n"
-      "mov x19, #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "tbz x8, #0, 19f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "b 19f\n"
       "18:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "19:"  // Height 1: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 22f\n"
       "20:"  // Height 1: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "b 22f\n"
       "21:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -240,176 +240,173 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "22:"  // Height 1: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "23:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 24f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "cbnz x14, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "cbnz x15, 25f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
       "b 25f\n"
       "24:"  // Height 1: setup direct input
-      "mov x12, %x[input_ptr]\n"
+      "mov x13, %x[input_ptr]\n"
       "25:"  // Height 1: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 28f\n"
-      "ldr q0, [x12, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
       "ldr q6, [x17, #0x0]\n"
-      "cmp x13, #0x10\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 27f\n"
       "26:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
-      "ldr x11, [x17, #0x18]\n"
-      "add x12, x12, #0x10\n"
       "ldr d6, [x17, #0x20]\n"
-      "sub x13, x13, #0x8\n"
-      "ldr x10, [x17, #0x28]\n"
-      "cmp x13, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x38]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "ldr d6, [x17, #0x40]\n"
-      "ldr x10, [x17, #0x48]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x58]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "ldr d6, [x17, #0x60]\n"
-      "ldr x10, [x17, #0x68]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x78]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "ldr d6, [x17, #0x80]\n"
-      "ldr x10, [x17, #0x88]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x98]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x98]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "ldr d6, [x17, #0xa0]\n"
-      "ldr x10, [x17, #0xa8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xb8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "ldr d6, [x17, #0xc0]\n"
-      "ldr x10, [x17, #0xc8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "ldr d6, [x17, #0xe0]\n"
-      "ldr x10, [x17, #0xe8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xf8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "ldr d6, [x17, #0x100]\n"
-      "ldr x10, [x17, #0x108]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x118]\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x118]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "ldr d6, [x17, #0x120]\n"
-      "ldr x10, [x17, #0x128]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x138]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x138]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "ldr d6, [x17, #0x140]\n"
-      "ldr x10, [x17, #0x148]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x158]\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x158]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "ldr d6, [x17, #0x160]\n"
-      "ldr x10, [x17, #0x168]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x178]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x178]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "ldr d6, [x17, #0x180]\n"
-      "ldr x10, [x17, #0x188]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x198]\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x198]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "ldr d6, [x17, #0x1a0]\n"
-      "ldr x10, [x17, #0x1a8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1b8]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x1b8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "ldr d6, [x17, #0x1c0]\n"
-      "ldr x10, [x17, #0x1c8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1d8]\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x1d8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "ldr d6, [x17, #0x1e0]\n"
-      "ldr x10, [x17, #0x1e8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1f8]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1f0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x1f8]\n"
+      "mov v7.d[1], x11\n"
+      "add x13, x13, #0x10\n"
       "add x17, x17, #0x200\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "ldr d6, [x17, #0x0]\n"
-      "ldr x10, [x17, #0x8]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d0, [x12, #0x0]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d0, [x13, #0x0]\n"
+      "sub x14, x14, #0x8\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x10\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "bge 26b\n"
       "27:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "ldr q6, [x17, #0x20]\n"
-      "sub x13, x13, #0x8\n"
-      "add x12, x12, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v10.8h, v6.8h, v0.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
+      "fmla v10.8h, v6.8h, v0.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "ldr q7, [x17, #0x50]\n"
@@ -465,321 +462,324 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ldr q6, [x17, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "ldr q7, [x17, #0x1f0]\n"
+      "add x13, x13, #0x10\n"
+      "sub x14, x14, #0x8\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
-      "add x17, x17, #0x200\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
+      "add x17, x17, #0x200\n"
       "28:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x13, 30f\n"
+      "cbz x14, 30f\n"
       "29:"  // Height 1: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q6, [x17, #0x20]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x30]\n"
-      "add x17, x17, #0x40\n"
+      "ldr q6, [x17, #0x20]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "cbnz x13, 29b\n"
+      "add x17, x17, #0x40\n"
+      "cbnz x14, 29b\n"
       "30:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 23b\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "tbz %x[flags], #1, 31f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
       "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
       "31:"  // Height 1: No activation
       "cmp x8, #0x20\n"
       "bge 48f\n"
       "tbz x8, #4, 39f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "tbz x8, #3, 35f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "tbz x8, #2, 33f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "tbz x8, #1, 32f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "b 47f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_28
       "tbz x8, #0, 47f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "b 47f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_24
       "tbz x8, #1, 34f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "b 47f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_24
       "tbz x8, #0, 47f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "b 47f\n"
       "35:"  // Height 1: Partial direct writeback: partial_4_16
       "tbz x8, #2, 37f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "tbz x8, #1, 36f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "b 47f\n"
       "36:"  // Height 1: Partial direct writeback: partial_1_20
       "tbz x8, #0, 47f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "b 47f\n"
       "37:"  // Height 1: Partial direct writeback: partial_2_16
       "tbz x8, #1, 38f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "b 47f\n"
       "38:"  // Height 1: Partial direct writeback: partial_1_16
       "tbz x8, #0, 47f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "b 47f\n"
       "39:"  // Height 1: Partial direct writeback: partial_8_0
       "tbz x8, #3, 43f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "tbz x8, #2, 41f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "tbz x8, #1, 40f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "b 47f\n"
       "40:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x8, #0, 47f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "b 47f\n"
       "41:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x8, #1, 42f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "b 47f\n"
       "42:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x8, #0, 47f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "b 47f\n"
       "43:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x8, #2, 45f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "tbz x8, #1, 44f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "b 47f\n"
       "44:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x8, #0, 47f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "b 47f\n"
       "45:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x8, #1, 46f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "tbz x8, #0, 47f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "b 47f\n"
       "46:"  // Height 1: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "47:"  // Height 1: Partial direct writeback: Done
       "b 49f\n"
       "48:"  // Height 1: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "49:"  // Height 1: Writeback done
       "subs x8, x8, #0x20\n"
       "bgt 2b\n"
       "b 296f\n"
       "50:"  // Height 2
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "51:"  // Height 2: Column loop
-      "cbz x16, 52f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 52f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
+      "add x7, x7, #0x40\n"
       "b 71f\n"
       "52:"  // Height 2: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x8, #0x20\n"
-      "add x25, x15, x19, LSL #1\n"
+      "add x25, x16, x20, LSL #1\n"
       "bge 69f\n"
       "tbz x8, #4, 60f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
       "ld1 { v13.8h }, [x25], #0x10\n"
       "tbz x8, #3, 56f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "ld1 { v14.8h }, [x25], #0x10\n"
       "tbz x8, #2, 54f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "ldr d15, [x25], #0x8\n"
       "tbz x8, #1, 53f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "ld1 { v15.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "ld1 { v15.h }[6], [x25]\n"
       "b 68f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "ld1 { v15.h }[4], [x25]\n"
       "b 68f\n"
       "54:"  // Height 2: Partial accumulate: partial_2_24
       "tbz x8, #1, 55f\n"
-      "ldr s11, [x15], #0x4\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "ldr s15, [x25], #0x4\n"
-      "mov x19, #0x34\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "ld1 { v15.h }[2], [x25]\n"
       "b 68f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 68f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "ldr h15, [x25, #0x0]\n"
       "b 68f\n"
       "56:"  // Height 2: Partial accumulate: partial_4_16
       "tbz x8, #2, 58f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "ldr d14, [x25], #0x8\n"
       "tbz x8, #1, 57f\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "ld1 { v14.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "ld1 { v14.h }[6], [x25]\n"
       "b 68f\n"
       "57:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "ld1 { v14.h }[4], [x25]\n"
       "b 68f\n"
       "58:"  // Height 2: Partial accumulate: partial_2_16
       "tbz x8, #1, 59f\n"
-      "ldr s10, [x15], #0x4\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "ldr s14, [x25], #0x4\n"
-      "mov x19, #0x24\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "ld1 { v14.h }[2], [x25]\n"
       "b 68f\n"
       "59:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 68f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "ldr h14, [x25, #0x0]\n"
       "b 68f\n"
       "60:"  // Height 2: Partial accumulate: partial_8_0
       "tbz x8, #3, 64f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "tbz x8, #2, 62f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "ldr d13, [x25], #0x8\n"
       "tbz x8, #1, 61f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "ld1 { v13.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "ld1 { v13.h }[6], [x25]\n"
       "b 68f\n"
       "61:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "ld1 { v13.h }[4], [x25]\n"
       "b 68f\n"
       "62:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x8, #1, 63f\n"
-      "ldr s9, [x15], #0x4\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "ldr s13, [x25], #0x4\n"
-      "mov x19, #0x14\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "ld1 { v13.h }[2], [x25]\n"
       "b 68f\n"
       "63:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 68f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "ldr h13, [x25, #0x0]\n"
       "b 68f\n"
       "64:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x8, #2, 66f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "ldr d12, [x25], #0x8\n"
       "tbz x8, #1, 65f\n"
-      "mov x19, #0xc\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "ld1 { v12.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "ld1 { v12.h }[6], [x25]\n"
       "b 68f\n"
       "65:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "ld1 { v12.h }[4], [x25]\n"
       "b 68f\n"
       "66:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x8, #1, 67f\n"
-      "ldr s8, [x15], #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "ldr s12, [x25], #0x4\n"
-      "mov x19, #0x4\n"
       "tbz x8, #0, 68f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "ld1 { v12.h }[2], [x25]\n"
       "b 68f\n"
       "67:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr h12, [x25, #0x0]\n"
       "68:"  // Height 2: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 71f\n"
       "69:"  // Height 2: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -795,227 +795,227 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "71:"  // Height 2: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "72:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "cbnz x14, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
-      "add x28, x28, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "cbnz x15, 74f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
+      "add x9, x9, x20, LSL #1\n"
       "b 74f\n"
       "73:"  // Height 2: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #1\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #1\n"
       "74:"  // Height 2: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 77f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x10\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
+      "ldr q1, [x9, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 76f\n"
       "75:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x18]\n"
       "ldr d6, [x17, #0x20]\n"
-      "add x12, x12, #0x10\n"
-      "ldr x10, [x17, #0x28]\n"
-      "add x28, x28, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "sub x13, x13, #0x8\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "ldr d6, [x17, #0x40]\n"
-      "ldr x10, [x17, #0x48]\n"
-      "cmp x13, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x58]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "ldr d6, [x17, #0x60]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x78]\n"
-      "ldr x27, [x28, #0x8]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "ldr d6, [x17, #0x80]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x98]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x98]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
       "ldr d6, [x17, #0xa0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
       "ldr d6, [x17, #0xc0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
       "ldr d6, [x17, #0xe0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0x108]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
       "ldr d6, [x17, #0x100]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x118]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x118]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x128]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
       "ldr d6, [x17, #0x120]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x138]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x138]\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x148]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
       "ldr d6, [x17, #0x140]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x158]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x158]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x168]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
       "ldr d6, [x17, #0x160]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x178]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x178]\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x188]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
       "ldr d6, [x17, #0x180]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x198]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x198]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1a8]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
       "ldr d6, [x17, #0x1a0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1b8]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x1b8]\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1c8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
       "ldr d6, [x17, #0x1c0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1d8]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x1d8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
-      "ldr x10, [x17, #0x1e8]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
       "ldr d6, [x17, #0x1e0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x1f8]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x1f8]\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "ldr d7, [x17, #0x1f0]\n"
-      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "mov v6.d[1], x12\n"
+      "add x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
       "add x17, x17, #0x200\n"
+      "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "ldr d6, [x17, #0x0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d1, [x28, #0x0]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
+      "ldr d1, [x9, #0x0]\n"
+      "sub x14, x14, #0x8\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x10\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x28, [x9, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "bge 75b\n"
       "76:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
-      "sub x13, x13, #0x8\n"
-      "add x12, x12, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x9, x9, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x8\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "fmla v11.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
+      "fmla v11.8h, v7.8h, v0.h[0]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x17, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "ldr q6, [x17, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
@@ -1099,14 +1099,14 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v11.8h, v7.8h, v0.h[7]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "77:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x13, 79f\n"
+      "cbz x14, 79f\n"
       "78:"  // Height 2: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
-      "ldr h1, [x28], #0x2\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
+      "ldr h1, [x9], #0x2\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
@@ -1117,21 +1117,19 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "cbnz x13, 78b\n"
+      "cbnz x14, 78b\n"
       "79:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 80f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x20]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
@@ -1140,149 +1138,151 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v13.8h, v13.8h, v0.8h\n"
       "fmin v14.8h, v14.8h, v0.8h\n"
       "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
       "80:"  // Height 2: No activation
       "cmp x8, #0x20\n"
       "bge 97f\n"
       "tbz x8, #4, 88f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v13.8h }, [x25], #0x10\n"
       "tbz x8, #3, 84f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "st1 { v14.8h }, [x25], #0x10\n"
       "tbz x8, #2, 82f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "tbz x8, #1, 81f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "st1 { v15.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "st1 { v15.h }[6], [x25]\n"
       "b 96f\n"
       "81:"  // Height 2: Partial direct writeback: partial_1_28
       "tbz x8, #0, 96f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "st1 { v15.h }[4], [x25]\n"
       "b 96f\n"
       "82:"  // Height 2: Partial direct writeback: partial_2_24
       "tbz x8, #1, 83f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "str s15, [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "st1 { v15.h }[2], [x25]\n"
       "b 96f\n"
       "83:"  // Height 2: Partial direct writeback: partial_1_24
       "tbz x8, #0, 96f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "str h15, [x25, #0x0]\n"
       "b 96f\n"
       "84:"  // Height 2: Partial direct writeback: partial_4_16
       "tbz x8, #2, 86f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "tbz x8, #1, 85f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "st1 { v14.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "st1 { v14.h }[6], [x25]\n"
       "b 96f\n"
       "85:"  // Height 2: Partial direct writeback: partial_1_20
       "tbz x8, #0, 96f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "st1 { v14.h }[4], [x25]\n"
       "b 96f\n"
       "86:"  // Height 2: Partial direct writeback: partial_2_16
       "tbz x8, #1, 87f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "str s14, [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "st1 { v14.h }[2], [x25]\n"
       "b 96f\n"
       "87:"  // Height 2: Partial direct writeback: partial_1_16
       "tbz x8, #0, 96f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "str h14, [x25, #0x0]\n"
       "b 96f\n"
       "88:"  // Height 2: Partial direct writeback: partial_8_0
       "tbz x8, #3, 92f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "tbz x8, #2, 90f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "tbz x8, #1, 89f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "st1 { v13.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "st1 { v13.h }[6], [x25]\n"
       "b 96f\n"
       "89:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x8, #0, 96f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "st1 { v13.h }[4], [x25]\n"
       "b 96f\n"
       "90:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x8, #1, 91f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "str s13, [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "st1 { v13.h }[2], [x25]\n"
       "b 96f\n"
       "91:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x8, #0, 96f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "str h13, [x25, #0x0]\n"
       "b 96f\n"
       "92:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x8, #2, 94f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "tbz x8, #1, 93f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "st1 { v12.s }[2], [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "st1 { v12.h }[6], [x25]\n"
       "b 96f\n"
       "93:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x8, #0, 96f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "st1 { v12.h }[4], [x25]\n"
       "b 96f\n"
       "94:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x8, #1, 95f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "str s12, [x25], #0x4\n"
       "tbz x8, #0, 96f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "st1 { v12.h }[2], [x25]\n"
       "b 96f\n"
       "95:"  // Height 2: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "str h12, [x25, #0x0]\n"
       "96:"  // Height 2: Partial direct writeback: Done
       "b 98f\n"
       "97:"  // Height 2: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -1292,213 +1292,213 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "bgt 51b\n"
       "b 296f\n"
       "99:"  // Height 3
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "100:"  // Height 3: Column loop
-      "cbz x16, 101f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 101f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 120f\n"
       "101:"  // Height 3: no bias
       "tbz %x[flags], #0, 119f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
       "cmp x8, #0x20\n"
-      "add x25, x15, x19, LSL #1\n"
-      "add x24, x25, x19, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "bge 118f\n"
       "tbz x8, #4, 109f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
       "ld1 { v13.8h }, [x25], #0x10\n"
       "ld1 { v17.8h }, [x24], #0x10\n"
       "tbz x8, #3, 105f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "ld1 { v14.8h }, [x25], #0x10\n"
       "ld1 { v18.8h }, [x24], #0x10\n"
       "tbz x8, #2, 103f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "tbz x8, #1, 102f\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
-      "mov x19, #0x3c\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "ld1 { v15.s }[2], [x25], #0x4\n"
       "ld1 { v19.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "ld1 { v15.h }[6], [x25]\n"
       "ld1 { v19.h }[6], [x24]\n"
       "b 117f\n"
       "102:"  // Height 3: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "ld1 { v15.h }[4], [x25]\n"
       "ld1 { v19.h }[4], [x24]\n"
       "b 117f\n"
       "103:"  // Height 3: Partial accumulate: partial_2_24
       "tbz x8, #1, 104f\n"
-      "ldr s11, [x15], #0x4\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "ldr s15, [x25], #0x4\n"
-      "mov x19, #0x34\n"
       "ldr s19, [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "ld1 { v15.h }[2], [x25]\n"
       "ld1 { v19.h }[2], [x24]\n"
       "b 117f\n"
       "104:"  // Height 3: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 117f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "ldr h15, [x25, #0x0]\n"
       "ldr h19, [x24, #0x0]\n"
       "b 117f\n"
       "105:"  // Height 3: Partial accumulate: partial_4_16
       "tbz x8, #2, 107f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "ldr d14, [x25], #0x8\n"
       "ldr d18, [x24], #0x8\n"
       "tbz x8, #1, 106f\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
-      "mov x19, #0x2c\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "ld1 { v14.s }[2], [x25], #0x4\n"
       "ld1 { v18.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "ld1 { v14.h }[6], [x25]\n"
       "ld1 { v18.h }[6], [x24]\n"
       "b 117f\n"
       "106:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "ld1 { v14.h }[4], [x25]\n"
       "ld1 { v18.h }[4], [x24]\n"
       "b 117f\n"
       "107:"  // Height 3: Partial accumulate: partial_2_16
       "tbz x8, #1, 108f\n"
-      "ldr s10, [x15], #0x4\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "ldr s14, [x25], #0x4\n"
-      "mov x19, #0x24\n"
       "ldr s18, [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "ld1 { v14.h }[2], [x25]\n"
       "ld1 { v18.h }[2], [x24]\n"
       "b 117f\n"
       "108:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 117f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "ldr h14, [x25, #0x0]\n"
       "ldr h18, [x24, #0x0]\n"
       "b 117f\n"
       "109:"  // Height 3: Partial accumulate: partial_8_0
       "tbz x8, #3, 113f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
       "tbz x8, #2, 111f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "tbz x8, #1, 110f\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
-      "mov x19, #0x1c\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "ld1 { v13.s }[2], [x25], #0x4\n"
       "ld1 { v17.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "ld1 { v13.h }[6], [x25]\n"
       "ld1 { v17.h }[6], [x24]\n"
       "b 117f\n"
       "110:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "ld1 { v13.h }[4], [x25]\n"
       "ld1 { v17.h }[4], [x24]\n"
       "b 117f\n"
       "111:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x8, #1, 112f\n"
-      "ldr s9, [x15], #0x4\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "ldr s13, [x25], #0x4\n"
-      "mov x19, #0x14\n"
       "ldr s17, [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "ld1 { v13.h }[2], [x25]\n"
       "ld1 { v17.h }[2], [x24]\n"
       "b 117f\n"
       "112:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 117f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "ldr h13, [x25, #0x0]\n"
       "ldr h17, [x24, #0x0]\n"
       "b 117f\n"
       "113:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x8, #2, 115f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "ldr d12, [x25], #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "tbz x8, #1, 114f\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
-      "mov x19, #0xc\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "ld1 { v12.s }[2], [x25], #0x4\n"
       "ld1 { v16.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "ld1 { v12.h }[6], [x25]\n"
       "ld1 { v16.h }[6], [x24]\n"
       "b 117f\n"
       "114:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "ld1 { v12.h }[4], [x25]\n"
       "ld1 { v16.h }[4], [x24]\n"
       "b 117f\n"
       "115:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x8, #1, 116f\n"
-      "ldr s8, [x15], #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "ldr s12, [x25], #0x4\n"
-      "mov x19, #0x4\n"
       "ldr s16, [x24], #0x4\n"
       "tbz x8, #0, 117f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "ld1 { v12.h }[2], [x25]\n"
       "ld1 { v16.h }[2], [x24]\n"
       "b 117f\n"
       "116:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr h12, [x25, #0x0]\n"
       "ldr h16, [x24, #0x0]\n"
       "117:"  // Height 3: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 120f\n"
       "118:"  // Height 3: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -1522,271 +1522,271 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "120:"  // Height 3: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "121:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 122f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "cbnz x14, 123f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
-      "add x28, x28, x19, LSL #1\n"
-      "add x26, x26, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "cbnz x15, 123f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
+      "add x9, x9, x20, LSL #1\n"
+      "add x27, x27, x20, LSL #1\n"
       "b 123f\n"
       "122:"  // Height 3: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #1\n"
-      "add x26, x28, x19, LSL #1\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #1\n"
+      "add x27, x9, x20, LSL #1\n"
       "123:"  // Height 3: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 126f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x10\n"
-      "ldr q2, [x26, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 125f\n"
       "124:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "ldr d6, [x17, #0x20]\n"
-      "ldr x10, [x17, #0x28]\n"
-      "add x12, x12, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x17, #0x38]\n"
-      "add x28, x28, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr x10, [x17, #0x48]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x40]\n"
-      "add x26, x26, #0x10\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x50]\n"
-      "sub x13, x13, #0x8\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x68]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x60]\n"
-      "cmp x13, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x88]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0x108]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0x118]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x100]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "ldr x11, [x17, #0x118]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x128]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x138]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x120]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "ldr x11, [x17, #0x138]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x148]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x158]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x140]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "ldr x11, [x17, #0x158]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x168]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x178]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x160]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "ldr x11, [x17, #0x178]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x188]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x198]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x180]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "ldr x11, [x17, #0x198]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1a8]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1b8]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x1a0]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "ldr x11, [x17, #0x1b8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1c8]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1d8]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x1c0]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "ldr x11, [x17, #0x1d8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
-      "ldr x10, [x17, #0x1e8]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
+      "ldr x11, [x17, #0x1f8]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "ldr x11, [x17, #0x1f8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
+      "add x13, x13, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x1f0]\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
+      "add x27, x27, #0x10\n"
       "add x17, x17, #0x200\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
-      "mov v0.d[1], x9\n"
-      "ldr d2, [x26, #0x0]\n"
-      "mov v1.d[1], x27\n"
-      "mov v2.d[1], x25\n"
+      "ldr d2, [x27, #0x0]\n"
+      "sub x14, x14, #0x8\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x10\n"
+      "ldr x26, [x27, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
+      "mov v2.d[1], x26\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "mov v7.d[1], x11\n"
       "bge 124b\n"
       "125:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x13, x13, #0x8\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "sub x14, x14, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "ldr q7, [x17, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
@@ -1901,15 +1901,15 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "126:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x13, 128f\n"
+      "cbz x14, 128f\n"
       "127:"  // Height 3: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
@@ -1924,23 +1924,21 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "cbnz x13, 127b\n"
+      "cbnz x14, 127b\n"
       "128:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 121b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #1\n"
       "prfm pstl1keep, [x24, #0x0]\n"
       "tbz %x[flags], #1, 129f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x20]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
@@ -1951,187 +1949,189 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v15.8h, v15.8h, v0.8h\n"
       "fmin v16.8h, v16.8h, v0.8h\n"
       "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
       "fmin v18.8h, v18.8h, v0.8h\n"
       "fmin v19.8h, v19.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
       "129:"  // Height 3: No activation
       "cmp x8, #0x20\n"
       "bge 146f\n"
       "tbz x8, #4, 137f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v13.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
       "st1 { v17.8h }, [x24], #0x10\n"
       "tbz x8, #3, 133f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "st1 { v14.8h }, [x25], #0x10\n"
       "st1 { v18.8h }, [x24], #0x10\n"
       "tbz x8, #2, 131f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "tbz x8, #1, 130f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "st1 { v15.s }[2], [x25], #0x4\n"
       "st1 { v19.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "st1 { v15.h }[6], [x25]\n"
       "st1 { v19.h }[6], [x24]\n"
       "b 145f\n"
       "130:"  // Height 3: Partial direct writeback: partial_1_28
       "tbz x8, #0, 145f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "st1 { v15.h }[4], [x25]\n"
       "st1 { v19.h }[4], [x24]\n"
       "b 145f\n"
       "131:"  // Height 3: Partial direct writeback: partial_2_24
       "tbz x8, #1, 132f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "str s15, [x25], #0x4\n"
       "str s19, [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "st1 { v15.h }[2], [x25]\n"
       "st1 { v19.h }[2], [x24]\n"
       "b 145f\n"
       "132:"  // Height 3: Partial direct writeback: partial_1_24
       "tbz x8, #0, 145f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "str h15, [x25, #0x0]\n"
       "str h19, [x24, #0x0]\n"
       "b 145f\n"
       "133:"  // Height 3: Partial direct writeback: partial_4_16
       "tbz x8, #2, 135f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "tbz x8, #1, 134f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "st1 { v14.s }[2], [x25], #0x4\n"
       "st1 { v18.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "st1 { v14.h }[6], [x25]\n"
       "st1 { v18.h }[6], [x24]\n"
       "b 145f\n"
       "134:"  // Height 3: Partial direct writeback: partial_1_20
       "tbz x8, #0, 145f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "st1 { v14.h }[4], [x25]\n"
       "st1 { v18.h }[4], [x24]\n"
       "b 145f\n"
       "135:"  // Height 3: Partial direct writeback: partial_2_16
       "tbz x8, #1, 136f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "str s14, [x25], #0x4\n"
       "str s18, [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "st1 { v14.h }[2], [x25]\n"
       "st1 { v18.h }[2], [x24]\n"
       "b 145f\n"
       "136:"  // Height 3: Partial direct writeback: partial_1_16
       "tbz x8, #0, 145f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "str h14, [x25, #0x0]\n"
       "str h18, [x24, #0x0]\n"
       "b 145f\n"
       "137:"  // Height 3: Partial direct writeback: partial_8_0
       "tbz x8, #3, 141f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
       "tbz x8, #2, 139f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "tbz x8, #1, 138f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "st1 { v13.s }[2], [x25], #0x4\n"
       "st1 { v17.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "st1 { v13.h }[6], [x25]\n"
       "st1 { v17.h }[6], [x24]\n"
       "b 145f\n"
       "138:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x8, #0, 145f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "st1 { v13.h }[4], [x25]\n"
       "st1 { v17.h }[4], [x24]\n"
       "b 145f\n"
       "139:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x8, #1, 140f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "str s13, [x25], #0x4\n"
       "str s17, [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "st1 { v13.h }[2], [x25]\n"
       "st1 { v17.h }[2], [x24]\n"
       "b 145f\n"
       "140:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x8, #0, 145f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "str h13, [x25, #0x0]\n"
       "str h17, [x24, #0x0]\n"
       "b 145f\n"
       "141:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x8, #2, 143f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "tbz x8, #1, 142f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "st1 { v12.s }[2], [x25], #0x4\n"
       "st1 { v16.s }[2], [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "st1 { v12.h }[6], [x25]\n"
       "st1 { v16.h }[6], [x24]\n"
       "b 145f\n"
       "142:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x8, #0, 145f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "st1 { v12.h }[4], [x25]\n"
       "st1 { v16.h }[4], [x24]\n"
       "b 145f\n"
       "143:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x8, #1, 144f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "str s12, [x25], #0x4\n"
       "str s16, [x24], #0x4\n"
       "tbz x8, #0, 145f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "st1 { v12.h }[2], [x25]\n"
       "st1 { v16.h }[2], [x24]\n"
       "b 145f\n"
       "144:"  // Height 3: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "str h12, [x25, #0x0]\n"
       "str h16, [x24, #0x0]\n"
       "145:"  // Height 3: Partial direct writeback: Done
       "b 147f\n"
       "146:"  // Height 3: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -2145,22 +2145,22 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "bgt 100b\n"
       "b 296f\n"
       "148:"  // Height 4
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "149:"  // Height 4: Column loop
-      "cbz x16, 150f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 150f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -2171,224 +2171,224 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 169f\n"
       "150:"  // Height 4: no bias
       "tbz %x[flags], #0, 168f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "cmp x8, #0x20\n"
-      "add x25, x15, x19, LSL #1\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "bge 167f\n"
       "tbz x8, #4, 158f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
       "ld1 { v13.8h }, [x25], #0x10\n"
       "ld1 { v17.8h }, [x24], #0x10\n"
-      "ld1 { v20.8h }, [x23], #0x10\n"
       "ld1 { v21.8h }, [x23], #0x10\n"
       "tbz x8, #3, 154f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "ld1 { v14.8h }, [x25], #0x10\n"
       "ld1 { v18.8h }, [x24], #0x10\n"
       "ld1 { v22.8h }, [x23], #0x10\n"
       "tbz x8, #2, 152f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "tbz x8, #1, 151f\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
-      "mov x19, #0x3c\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "ld1 { v15.s }[2], [x25], #0x4\n"
       "ld1 { v19.s }[2], [x24], #0x4\n"
       "ld1 { v23.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "ld1 { v15.h }[6], [x25]\n"
       "ld1 { v19.h }[6], [x24]\n"
       "ld1 { v23.h }[6], [x23]\n"
       "b 166f\n"
       "151:"  // Height 4: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "ld1 { v15.h }[4], [x25]\n"
       "ld1 { v19.h }[4], [x24]\n"
       "ld1 { v23.h }[4], [x23]\n"
       "b 166f\n"
       "152:"  // Height 4: Partial accumulate: partial_2_24
       "tbz x8, #1, 153f\n"
-      "ldr s11, [x15], #0x4\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "ldr s15, [x25], #0x4\n"
-      "mov x19, #0x34\n"
       "ldr s19, [x24], #0x4\n"
       "ldr s23, [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "ld1 { v15.h }[2], [x25]\n"
       "ld1 { v19.h }[2], [x24]\n"
       "ld1 { v23.h }[2], [x23]\n"
       "b 166f\n"
       "153:"  // Height 4: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 166f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "ldr h15, [x25, #0x0]\n"
       "ldr h19, [x24, #0x0]\n"
       "ldr h23, [x23, #0x0]\n"
       "b 166f\n"
       "154:"  // Height 4: Partial accumulate: partial_4_16
       "tbz x8, #2, 156f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "ldr d14, [x25], #0x8\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "tbz x8, #1, 155f\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
-      "mov x19, #0x2c\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "ld1 { v14.s }[2], [x25], #0x4\n"
       "ld1 { v18.s }[2], [x24], #0x4\n"
       "ld1 { v22.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "ld1 { v14.h }[6], [x25]\n"
       "ld1 { v18.h }[6], [x24]\n"
       "ld1 { v22.h }[6], [x23]\n"
       "b 166f\n"
       "155:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "ld1 { v14.h }[4], [x25]\n"
       "ld1 { v18.h }[4], [x24]\n"
       "ld1 { v22.h }[4], [x23]\n"
       "b 166f\n"
       "156:"  // Height 4: Partial accumulate: partial_2_16
       "tbz x8, #1, 157f\n"
-      "ldr s10, [x15], #0x4\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "ldr s14, [x25], #0x4\n"
-      "mov x19, #0x24\n"
       "ldr s18, [x24], #0x4\n"
       "ldr s22, [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "ld1 { v14.h }[2], [x25]\n"
       "ld1 { v18.h }[2], [x24]\n"
       "ld1 { v22.h }[2], [x23]\n"
       "b 166f\n"
       "157:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 166f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "ldr h14, [x25, #0x0]\n"
       "ldr h18, [x24, #0x0]\n"
       "ldr h22, [x23, #0x0]\n"
       "b 166f\n"
       "158:"  // Height 4: Partial accumulate: partial_8_0
       "tbz x8, #3, 162f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
       "ld1 { v20.8h }, [x23], #0x10\n"
       "tbz x8, #2, 160f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "tbz x8, #1, 159f\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
-      "mov x19, #0x1c\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "ld1 { v13.s }[2], [x25], #0x4\n"
       "ld1 { v17.s }[2], [x24], #0x4\n"
       "ld1 { v21.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "ld1 { v13.h }[6], [x25]\n"
       "ld1 { v17.h }[6], [x24]\n"
       "ld1 { v21.h }[6], [x23]\n"
       "b 166f\n"
       "159:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "ld1 { v13.h }[4], [x25]\n"
       "ld1 { v17.h }[4], [x24]\n"
       "ld1 { v21.h }[4], [x23]\n"
       "b 166f\n"
       "160:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x8, #1, 161f\n"
-      "ldr s9, [x15], #0x4\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "ldr s13, [x25], #0x4\n"
-      "mov x19, #0x14\n"
       "ldr s17, [x24], #0x4\n"
       "ldr s21, [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "ld1 { v13.h }[2], [x25]\n"
       "ld1 { v17.h }[2], [x24]\n"
       "ld1 { v21.h }[2], [x23]\n"
       "b 166f\n"
       "161:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 166f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "ldr h13, [x25, #0x0]\n"
       "ldr h17, [x24, #0x0]\n"
       "ldr h21, [x23, #0x0]\n"
       "b 166f\n"
       "162:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x8, #2, 164f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "ldr d12, [x25], #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "tbz x8, #1, 163f\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
-      "mov x19, #0xc\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "ld1 { v12.s }[2], [x25], #0x4\n"
       "ld1 { v16.s }[2], [x24], #0x4\n"
       "ld1 { v20.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "ld1 { v12.h }[6], [x25]\n"
       "ld1 { v16.h }[6], [x24]\n"
       "ld1 { v20.h }[6], [x23]\n"
       "b 166f\n"
       "163:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "ld1 { v12.h }[4], [x25]\n"
       "ld1 { v16.h }[4], [x24]\n"
       "ld1 { v20.h }[4], [x23]\n"
       "b 166f\n"
       "164:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x8, #1, 165f\n"
-      "ldr s8, [x15], #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "ldr s12, [x25], #0x4\n"
-      "mov x19, #0x4\n"
       "ldr s16, [x24], #0x4\n"
       "ldr s20, [x23], #0x4\n"
       "tbz x8, #0, 166f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "ld1 { v12.h }[2], [x25]\n"
       "ld1 { v16.h }[2], [x24]\n"
       "ld1 { v20.h }[2], [x23]\n"
       "b 166f\n"
       "165:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr h12, [x25, #0x0]\n"
       "ldr h16, [x24, #0x0]\n"
       "ldr h20, [x23, #0x0]\n"
       "166:"  // Height 4: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 169f\n"
       "167:"  // Height 4: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -2420,316 +2420,316 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "169:"  // Height 4: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "170:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 171f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x14, 172f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
-      "add x28, x28, x19, LSL #1\n"
-      "add x26, x26, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x15, 172f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
+      "add x9, x9, x20, LSL #1\n"
+      "add x27, x27, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 172f\n"
       "171:"  // Height 4: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #1\n"
-      "add x26, x28, x19, LSL #1\n"
-      "add x24, x26, x19, LSL #1\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #1\n"
+      "add x27, x9, x20, LSL #1\n"
+      "add x25, x27, x20, LSL #1\n"
       "172:"  // Height 4: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 175f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x10\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 174f\n"
       "173:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
       "ldr d6, [x17, #0x20]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x38]\n"
-      "add x12, x12, #0x10\n"
-      "add x28, x28, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x9, x9, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "ldr x10, [x17, #0x48]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x68]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "add x26, x26, #0x10\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "fmla v9.8h, v7.8h, v0.h[1]\n"
       "ldr d6, [x17, #0x60]\n"
+      "fmla v9.8h, v7.8h, v0.h[1]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
-      "add x24, x24, #0x10\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
-      "sub x13, x13, #0x8\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x8\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
-      "cmp x13, #0x10\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
+      "cmp x14, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
-      "ldr x10, [x17, #0x108]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0x118]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
       "ldr d6, [x17, #0x100]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
-      "ldr x11, [x17, #0x118]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x128]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x138]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
       "ldr d6, [x17, #0x120]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
-      "ldr x11, [x17, #0x138]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
-      "ldr x10, [x17, #0x148]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x158]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
       "ldr d6, [x17, #0x140]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
-      "ldr x11, [x17, #0x158]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x168]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x178]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
       "ldr d6, [x17, #0x160]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
-      "ldr x11, [x17, #0x178]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
-      "ldr x10, [x17, #0x188]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x198]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
       "ldr d6, [x17, #0x180]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
-      "ldr x11, [x17, #0x198]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1a8]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1b8]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
       "ldr d6, [x17, #0x1a0]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
-      "ldr x11, [x17, #0x1b8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
-      "ldr x10, [x17, #0x1c8]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1d8]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
       "ldr d6, [x17, #0x1c0]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
-      "ldr x11, [x17, #0x1d8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
-      "ldr x10, [x17, #0x1e8]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
+      "ldr x11, [x17, #0x1f8]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
       "ldr d6, [x17, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
-      "ldr x11, [x17, #0x1f8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "ldr d7, [x17, #0x1f0]\n"
-      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x200\n"
+      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
+      "ldr d3, [x25, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v7.d[1], x11\n"
       "bge 173b\n"
       "174:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x13, x13, #0x8\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "fmla v11.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
+      "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
@@ -2874,16 +2874,16 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "175:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x13, 177f\n"
+      "cbz x14, 177f\n"
       "176:"  // Height 4: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
@@ -2902,25 +2902,23 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "cbnz x13, 176b\n"
+      "cbnz x14, 176b\n"
       "177:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 170b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #1\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 178f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x20]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
@@ -2931,34 +2929,36 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v15.8h, v15.8h, v0.8h\n"
       "fmin v16.8h, v16.8h, v0.8h\n"
       "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
       "fmin v18.8h, v18.8h, v0.8h\n"
       "fmin v19.8h, v19.8h, v0.8h\n"
       "fmin v20.8h, v20.8h, v0.8h\n"
       "fmin v21.8h, v21.8h, v0.8h\n"
       "fmin v22.8h, v22.8h, v0.8h\n"
       "fmin v23.8h, v23.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
       "178:"  // Height 4: No activation
       "cmp x8, #0x20\n"
       "bge 195f\n"
       "tbz x8, #4, 186f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v13.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
@@ -2966,192 +2966,192 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "st1 { v20.8h }, [x23], #0x10\n"
       "st1 { v21.8h }, [x23], #0x10\n"
       "tbz x8, #3, 182f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "st1 { v14.8h }, [x25], #0x10\n"
       "st1 { v18.8h }, [x24], #0x10\n"
       "st1 { v22.8h }, [x23], #0x10\n"
       "tbz x8, #2, 180f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "tbz x8, #1, 179f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "st1 { v15.s }[2], [x25], #0x4\n"
       "st1 { v19.s }[2], [x24], #0x4\n"
       "st1 { v23.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "st1 { v15.h }[6], [x25]\n"
       "st1 { v19.h }[6], [x24]\n"
       "st1 { v23.h }[6], [x23]\n"
       "b 194f\n"
       "179:"  // Height 4: Partial direct writeback: partial_1_28
       "tbz x8, #0, 194f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "st1 { v15.h }[4], [x25]\n"
       "st1 { v19.h }[4], [x24]\n"
       "st1 { v23.h }[4], [x23]\n"
       "b 194f\n"
       "180:"  // Height 4: Partial direct writeback: partial_2_24
       "tbz x8, #1, 181f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "str s15, [x25], #0x4\n"
       "str s19, [x24], #0x4\n"
       "str s23, [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "st1 { v15.h }[2], [x25]\n"
       "st1 { v19.h }[2], [x24]\n"
       "st1 { v23.h }[2], [x23]\n"
       "b 194f\n"
       "181:"  // Height 4: Partial direct writeback: partial_1_24
       "tbz x8, #0, 194f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "str h15, [x25, #0x0]\n"
       "str h19, [x24, #0x0]\n"
       "str h23, [x23, #0x0]\n"
       "b 194f\n"
       "182:"  // Height 4: Partial direct writeback: partial_4_16
       "tbz x8, #2, 184f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "tbz x8, #1, 183f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "st1 { v14.s }[2], [x25], #0x4\n"
       "st1 { v18.s }[2], [x24], #0x4\n"
       "st1 { v22.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "st1 { v14.h }[6], [x25]\n"
       "st1 { v18.h }[6], [x24]\n"
       "st1 { v22.h }[6], [x23]\n"
       "b 194f\n"
       "183:"  // Height 4: Partial direct writeback: partial_1_20
       "tbz x8, #0, 194f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "st1 { v14.h }[4], [x25]\n"
       "st1 { v18.h }[4], [x24]\n"
       "st1 { v22.h }[4], [x23]\n"
       "b 194f\n"
       "184:"  // Height 4: Partial direct writeback: partial_2_16
       "tbz x8, #1, 185f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "str s14, [x25], #0x4\n"
       "str s18, [x24], #0x4\n"
       "str s22, [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "st1 { v14.h }[2], [x25]\n"
       "st1 { v18.h }[2], [x24]\n"
       "st1 { v22.h }[2], [x23]\n"
       "b 194f\n"
       "185:"  // Height 4: Partial direct writeback: partial_1_16
       "tbz x8, #0, 194f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "str h14, [x25, #0x0]\n"
       "str h18, [x24, #0x0]\n"
       "str h22, [x23, #0x0]\n"
       "b 194f\n"
       "186:"  // Height 4: Partial direct writeback: partial_8_0
       "tbz x8, #3, 190f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
       "st1 { v20.8h }, [x23], #0x10\n"
       "tbz x8, #2, 188f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "tbz x8, #1, 187f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "st1 { v13.s }[2], [x25], #0x4\n"
       "st1 { v17.s }[2], [x24], #0x4\n"
       "st1 { v21.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "st1 { v13.h }[6], [x25]\n"
       "st1 { v17.h }[6], [x24]\n"
       "st1 { v21.h }[6], [x23]\n"
       "b 194f\n"
       "187:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x8, #0, 194f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "st1 { v13.h }[4], [x25]\n"
       "st1 { v17.h }[4], [x24]\n"
       "st1 { v21.h }[4], [x23]\n"
       "b 194f\n"
       "188:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x8, #1, 189f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "str s13, [x25], #0x4\n"
       "str s17, [x24], #0x4\n"
       "str s21, [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "st1 { v13.h }[2], [x25]\n"
       "st1 { v17.h }[2], [x24]\n"
       "st1 { v21.h }[2], [x23]\n"
       "b 194f\n"
       "189:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x8, #0, 194f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "str h13, [x25, #0x0]\n"
       "str h17, [x24, #0x0]\n"
       "str h21, [x23, #0x0]\n"
       "b 194f\n"
       "190:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x8, #2, 192f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "tbz x8, #1, 191f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "st1 { v12.s }[2], [x25], #0x4\n"
       "st1 { v16.s }[2], [x24], #0x4\n"
       "st1 { v20.s }[2], [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "st1 { v12.h }[6], [x25]\n"
       "st1 { v16.h }[6], [x24]\n"
       "st1 { v20.h }[6], [x23]\n"
       "b 194f\n"
       "191:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x8, #0, 194f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "st1 { v12.h }[4], [x25]\n"
       "st1 { v16.h }[4], [x24]\n"
       "st1 { v20.h }[4], [x23]\n"
       "b 194f\n"
       "192:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x8, #1, 193f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "str s12, [x25], #0x4\n"
       "str s16, [x24], #0x4\n"
       "str s20, [x23], #0x4\n"
       "tbz x8, #0, 194f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "st1 { v12.h }[2], [x25]\n"
       "st1 { v16.h }[2], [x24]\n"
       "st1 { v20.h }[2], [x23]\n"
       "b 194f\n"
       "193:"  // Height 4: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "str h12, [x25, #0x0]\n"
       "str h16, [x24, #0x0]\n"
       "str h20, [x23, #0x0]\n"
       "194:"  // Height 4: Partial direct writeback: Done
       "b 196f\n"
       "195:"  // Height 4: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -3169,22 +3169,22 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "bgt 149b\n"
       "b 296f\n"
       "197:"  // Height 5
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "198:"  // Height 5: Column loop
-      "cbz x16, 199f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 199f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -3199,54 +3199,54 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 218f\n"
       "199:"  // Height 5: no bias
       "tbz %x[flags], #0, 217f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "cmp x8, #0x20\n"
-      "add x25, x15, x19, LSL #1\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "bge 216f\n"
       "tbz x8, #4, 207f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
-      "ld1 { v13.8h }, [x25], #0x10\n"
-      "ld1 { v17.8h }, [x24], #0x10\n"
       "ld1 { v20.8h }, [x23], #0x10\n"
       "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
       "ld1 { v21.8h }, [x23], #0x10\n"
       "ld1 { v25.8h }, [x22], #0x10\n"
       "tbz x8, #3, 203f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "ld1 { v14.8h }, [x25], #0x10\n"
       "ld1 { v18.8h }, [x24], #0x10\n"
       "ld1 { v22.8h }, [x23], #0x10\n"
       "ld1 { v26.8h }, [x22], #0x10\n"
       "tbz x8, #2, 201f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "ldr d27, [x22], #0x8\n"
       "tbz x8, #1, 200f\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
-      "mov x19, #0x3c\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "ld1 { v15.s }[2], [x25], #0x4\n"
       "ld1 { v19.s }[2], [x24], #0x4\n"
       "ld1 { v23.s }[2], [x23], #0x4\n"
       "ld1 { v27.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "ld1 { v15.h }[6], [x25]\n"
       "ld1 { v19.h }[6], [x24]\n"
       "ld1 { v23.h }[6], [x23]\n"
       "ld1 { v27.h }[6], [x22]\n"
       "b 215f\n"
       "200:"  // Height 5: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "ld1 { v15.h }[4], [x25]\n"
       "ld1 { v19.h }[4], [x24]\n"
       "ld1 { v23.h }[4], [x23]\n"
@@ -3254,23 +3254,23 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "201:"  // Height 5: Partial accumulate: partial_2_24
       "tbz x8, #1, 202f\n"
-      "ldr s11, [x15], #0x4\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "ldr s15, [x25], #0x4\n"
-      "mov x19, #0x34\n"
       "ldr s19, [x24], #0x4\n"
       "ldr s23, [x23], #0x4\n"
       "ldr s27, [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "ld1 { v15.h }[2], [x25]\n"
       "ld1 { v19.h }[2], [x24]\n"
       "ld1 { v23.h }[2], [x23]\n"
       "ld1 { v27.h }[2], [x22]\n"
       "b 215f\n"
       "202:"  // Height 5: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 215f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "ldr h15, [x25, #0x0]\n"
       "ldr h19, [x24, #0x0]\n"
       "ldr h23, [x23, #0x0]\n"
@@ -3278,29 +3278,29 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "203:"  // Height 5: Partial accumulate: partial_4_16
       "tbz x8, #2, 205f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "ldr d14, [x25], #0x8\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "ldr d26, [x22], #0x8\n"
       "tbz x8, #1, 204f\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
-      "mov x19, #0x2c\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "ld1 { v14.s }[2], [x25], #0x4\n"
       "ld1 { v18.s }[2], [x24], #0x4\n"
       "ld1 { v22.s }[2], [x23], #0x4\n"
       "ld1 { v26.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "ld1 { v14.h }[6], [x25]\n"
       "ld1 { v18.h }[6], [x24]\n"
       "ld1 { v22.h }[6], [x23]\n"
       "ld1 { v26.h }[6], [x22]\n"
       "b 215f\n"
       "204:"  // Height 5: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "ld1 { v14.h }[4], [x25]\n"
       "ld1 { v18.h }[4], [x24]\n"
       "ld1 { v22.h }[4], [x23]\n"
@@ -3308,23 +3308,23 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "205:"  // Height 5: Partial accumulate: partial_2_16
       "tbz x8, #1, 206f\n"
-      "ldr s10, [x15], #0x4\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "ldr s14, [x25], #0x4\n"
-      "mov x19, #0x24\n"
       "ldr s18, [x24], #0x4\n"
       "ldr s22, [x23], #0x4\n"
       "ldr s26, [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "ld1 { v14.h }[2], [x25]\n"
       "ld1 { v18.h }[2], [x24]\n"
       "ld1 { v22.h }[2], [x23]\n"
       "ld1 { v26.h }[2], [x22]\n"
       "b 215f\n"
       "206:"  // Height 5: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 215f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "ldr h14, [x25, #0x0]\n"
       "ldr h18, [x24, #0x0]\n"
       "ldr h22, [x23, #0x0]\n"
@@ -3332,35 +3332,35 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "207:"  // Height 5: Partial accumulate: partial_8_0
       "tbz x8, #3, 211f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
       "ld1 { v20.8h }, [x23], #0x10\n"
       "ld1 { v24.8h }, [x22], #0x10\n"
       "tbz x8, #2, 209f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "tbz x8, #1, 208f\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
-      "mov x19, #0x1c\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "ld1 { v13.s }[2], [x25], #0x4\n"
       "ld1 { v17.s }[2], [x24], #0x4\n"
       "ld1 { v21.s }[2], [x23], #0x4\n"
       "ld1 { v25.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "ld1 { v13.h }[6], [x25]\n"
       "ld1 { v17.h }[6], [x24]\n"
       "ld1 { v21.h }[6], [x23]\n"
       "ld1 { v25.h }[6], [x22]\n"
       "b 215f\n"
       "208:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "ld1 { v13.h }[4], [x25]\n"
       "ld1 { v17.h }[4], [x24]\n"
       "ld1 { v21.h }[4], [x23]\n"
@@ -3368,23 +3368,23 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "209:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x8, #1, 210f\n"
-      "ldr s9, [x15], #0x4\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "ldr s13, [x25], #0x4\n"
-      "mov x19, #0x14\n"
       "ldr s17, [x24], #0x4\n"
       "ldr s21, [x23], #0x4\n"
       "ldr s25, [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "ld1 { v13.h }[2], [x25]\n"
       "ld1 { v17.h }[2], [x24]\n"
       "ld1 { v21.h }[2], [x23]\n"
       "ld1 { v25.h }[2], [x22]\n"
       "b 215f\n"
       "210:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 215f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "ldr h13, [x25, #0x0]\n"
       "ldr h17, [x24, #0x0]\n"
       "ldr h21, [x23, #0x0]\n"
@@ -3392,29 +3392,29 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "211:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x8, #2, 213f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "ldr d12, [x25], #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "ldr d24, [x22], #0x8\n"
       "tbz x8, #1, 212f\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
-      "mov x19, #0xc\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "ld1 { v12.s }[2], [x25], #0x4\n"
       "ld1 { v16.s }[2], [x24], #0x4\n"
       "ld1 { v20.s }[2], [x23], #0x4\n"
       "ld1 { v24.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "ld1 { v12.h }[6], [x25]\n"
       "ld1 { v16.h }[6], [x24]\n"
       "ld1 { v20.h }[6], [x23]\n"
       "ld1 { v24.h }[6], [x22]\n"
       "b 215f\n"
       "212:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "ld1 { v12.h }[4], [x25]\n"
       "ld1 { v16.h }[4], [x24]\n"
       "ld1 { v20.h }[4], [x23]\n"
@@ -3422,34 +3422,34 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 215f\n"
       "213:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x8, #1, 214f\n"
-      "ldr s8, [x15], #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "ldr s12, [x25], #0x4\n"
-      "mov x19, #0x4\n"
       "ldr s16, [x24], #0x4\n"
       "ldr s20, [x23], #0x4\n"
       "ldr s24, [x22], #0x4\n"
       "tbz x8, #0, 215f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "ld1 { v12.h }[2], [x25]\n"
       "ld1 { v16.h }[2], [x24]\n"
       "ld1 { v20.h }[2], [x23]\n"
       "ld1 { v24.h }[2], [x22]\n"
       "b 215f\n"
       "214:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr h12, [x25, #0x0]\n"
       "ldr h16, [x24, #0x0]\n"
       "ldr h20, [x23, #0x0]\n"
       "ldr h24, [x22, #0x0]\n"
       "215:"  // Height 5: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 218f\n"
       "216:"  // Height 5: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -3489,359 +3489,359 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "218:"  // Height 5: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "219:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 220f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "cbnz x14, 221f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
-      "add x28, x28, x19, LSL #1\n"
-      "add x26, x26, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "cbnz x15, 221f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
+      "add x9, x9, x20, LSL #1\n"
+      "add x27, x27, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 221f\n"
       "220:"  // Height 5: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #1\n"
-      "add x26, x28, x19, LSL #1\n"
-      "add x24, x26, x19, LSL #1\n"
-      "add x22, x24, x19, LSL #1\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #1\n"
+      "add x27, x9, x20, LSL #1\n"
+      "add x25, x27, x20, LSL #1\n"
+      "add x23, x25, x20, LSL #1\n"
       "221:"  // Height 5: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 224f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x10\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 223f\n"
       "222:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "add x28, x28, #0x10\n"
+      "ldr d6, [x17, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "ldr d6, [x17, #0x20]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "add x27, x27, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x30]\n"
-      "add x26, x26, #0x10\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "ldr x10, [x17, #0x48]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x23, x23, #0x10\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "add x24, x24, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
       "ldr d6, [x17, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "add x22, x22, #0x10\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "sub x14, x14, #0x8\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "cmp x14, #0x10\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "ldr x10, [x17, #0x88]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "sub x13, x13, #0x8\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
-      "cmp x13, #0x10\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
-      "ldr x10, [x17, #0xa8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
-      "ldr x10, [x17, #0x108]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0x118]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
-      "ldr x11, [x17, #0x118]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
       "ldr d6, [x17, #0x100]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
-      "ldr x10, [x17, #0x128]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x138]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
-      "ldr x11, [x17, #0x138]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
       "ldr d6, [x17, #0x120]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
-      "ldr x10, [x17, #0x148]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x158]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
-      "ldr x11, [x17, #0x158]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
       "ldr d6, [x17, #0x140]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
-      "ldr x10, [x17, #0x168]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x178]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
-      "ldr x11, [x17, #0x178]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
       "ldr d6, [x17, #0x160]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
-      "ldr x10, [x17, #0x188]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x198]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
-      "ldr x11, [x17, #0x198]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
       "ldr d6, [x17, #0x180]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
-      "ldr x10, [x17, #0x1a8]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1b8]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
-      "ldr x11, [x17, #0x1b8]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
       "ldr d6, [x17, #0x1a0]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
-      "ldr x10, [x17, #0x1c8]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1d8]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
-      "ldr x11, [x17, #0x1d8]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
       "ldr d6, [x17, #0x1c0]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
-      "ldr x10, [x17, #0x1e8]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
+      "ldr x11, [x17, #0x1f8]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
-      "ldr x11, [x17, #0x1f8]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
       "ldr d6, [x17, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
       "ldr d7, [x17, #0x1f0]\n"
-      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x200\n"
+      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "fmla v26.8h, v6.8h, v4.h[7]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
+      "ldr d4, [x23, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v7.d[1], x11\n"
       "bge 222b\n"
       "223:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x13, x13, #0x8\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "sub x14, x14, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
@@ -4018,17 +4018,17 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "224:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x13, 226f\n"
+      "cbz x14, 226f\n"
       "225:"  // Height 5: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
@@ -4051,27 +4051,25 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "cbnz x13, 225b\n"
+      "cbnz x14, 225b\n"
       "226:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 219b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #1\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #1\n"
       "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 227f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x20]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
@@ -4082,16 +4080,6 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v15.8h, v15.8h, v0.8h\n"
       "fmin v16.8h, v16.8h, v0.8h\n"
       "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
       "fmin v18.8h, v18.8h, v0.8h\n"
       "fmin v19.8h, v19.8h, v0.8h\n"
       "fmin v20.8h, v20.8h, v0.8h\n"
@@ -4102,22 +4090,34 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v25.8h, v25.8h, v0.8h\n"
       "fmin v26.8h, v26.8h, v0.8h\n"
       "fmin v27.8h, v27.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
-      "fmax v24.8h, v24.8h, v1.8h\n"
-      "fmax v25.8h, v25.8h, v1.8h\n"
-      "fmax v26.8h, v26.8h, v1.8h\n"
-      "fmax v27.8h, v27.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
+      "fmax v24.8h, v24.8h, v0.8h\n"
+      "fmax v25.8h, v25.8h, v0.8h\n"
+      "fmax v26.8h, v26.8h, v0.8h\n"
+      "fmax v27.8h, v27.8h, v0.8h\n"
       "227:"  // Height 5: No activation
       "cmp x8, #0x20\n"
       "bge 244f\n"
       "tbz x8, #4, 235f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v13.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
@@ -4127,25 +4127,25 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "st1 { v24.8h }, [x22], #0x10\n"
       "st1 { v25.8h }, [x22], #0x10\n"
       "tbz x8, #3, 231f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "st1 { v14.8h }, [x25], #0x10\n"
       "st1 { v18.8h }, [x24], #0x10\n"
       "st1 { v22.8h }, [x23], #0x10\n"
       "st1 { v26.8h }, [x22], #0x10\n"
       "tbz x8, #2, 229f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "str d27, [x22], #0x8\n"
       "tbz x8, #1, 228f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "st1 { v15.s }[2], [x25], #0x4\n"
       "st1 { v19.s }[2], [x24], #0x4\n"
       "st1 { v23.s }[2], [x23], #0x4\n"
       "st1 { v27.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "st1 { v15.h }[6], [x25]\n"
       "st1 { v19.h }[6], [x24]\n"
       "st1 { v23.h }[6], [x23]\n"
@@ -4153,7 +4153,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "228:"  // Height 5: Partial direct writeback: partial_1_28
       "tbz x8, #0, 243f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "st1 { v15.h }[4], [x25]\n"
       "st1 { v19.h }[4], [x24]\n"
       "st1 { v23.h }[4], [x23]\n"
@@ -4161,13 +4161,13 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "229:"  // Height 5: Partial direct writeback: partial_2_24
       "tbz x8, #1, 230f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "str s15, [x25], #0x4\n"
       "str s19, [x24], #0x4\n"
       "str s23, [x23], #0x4\n"
       "str s27, [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "st1 { v15.h }[2], [x25]\n"
       "st1 { v19.h }[2], [x24]\n"
       "st1 { v23.h }[2], [x23]\n"
@@ -4175,7 +4175,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "230:"  // Height 5: Partial direct writeback: partial_1_24
       "tbz x8, #0, 243f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "str h15, [x25, #0x0]\n"
       "str h19, [x24, #0x0]\n"
       "str h23, [x23, #0x0]\n"
@@ -4183,19 +4183,19 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "231:"  // Height 5: Partial direct writeback: partial_4_16
       "tbz x8, #2, 233f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "str d26, [x22], #0x8\n"
       "tbz x8, #1, 232f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "st1 { v14.s }[2], [x25], #0x4\n"
       "st1 { v18.s }[2], [x24], #0x4\n"
       "st1 { v22.s }[2], [x23], #0x4\n"
       "st1 { v26.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "st1 { v14.h }[6], [x25]\n"
       "st1 { v18.h }[6], [x24]\n"
       "st1 { v22.h }[6], [x23]\n"
@@ -4203,7 +4203,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "232:"  // Height 5: Partial direct writeback: partial_1_20
       "tbz x8, #0, 243f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "st1 { v14.h }[4], [x25]\n"
       "st1 { v18.h }[4], [x24]\n"
       "st1 { v22.h }[4], [x23]\n"
@@ -4211,13 +4211,13 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "233:"  // Height 5: Partial direct writeback: partial_2_16
       "tbz x8, #1, 234f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "str s14, [x25], #0x4\n"
       "str s18, [x24], #0x4\n"
       "str s22, [x23], #0x4\n"
       "str s26, [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "st1 { v14.h }[2], [x25]\n"
       "st1 { v18.h }[2], [x24]\n"
       "st1 { v22.h }[2], [x23]\n"
@@ -4225,7 +4225,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "234:"  // Height 5: Partial direct writeback: partial_1_16
       "tbz x8, #0, 243f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "str h14, [x25, #0x0]\n"
       "str h18, [x24, #0x0]\n"
       "str h22, [x23, #0x0]\n"
@@ -4233,25 +4233,25 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "235:"  // Height 5: Partial direct writeback: partial_8_0
       "tbz x8, #3, 239f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
       "st1 { v20.8h }, [x23], #0x10\n"
       "st1 { v24.8h }, [x22], #0x10\n"
       "tbz x8, #2, 237f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "tbz x8, #1, 236f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "st1 { v13.s }[2], [x25], #0x4\n"
       "st1 { v17.s }[2], [x24], #0x4\n"
       "st1 { v21.s }[2], [x23], #0x4\n"
       "st1 { v25.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "st1 { v13.h }[6], [x25]\n"
       "st1 { v17.h }[6], [x24]\n"
       "st1 { v21.h }[6], [x23]\n"
@@ -4259,7 +4259,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "236:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x8, #0, 243f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "st1 { v13.h }[4], [x25]\n"
       "st1 { v17.h }[4], [x24]\n"
       "st1 { v21.h }[4], [x23]\n"
@@ -4267,13 +4267,13 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "237:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x8, #1, 238f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "str s13, [x25], #0x4\n"
       "str s17, [x24], #0x4\n"
       "str s21, [x23], #0x4\n"
       "str s25, [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "st1 { v13.h }[2], [x25]\n"
       "st1 { v17.h }[2], [x24]\n"
       "st1 { v21.h }[2], [x23]\n"
@@ -4281,7 +4281,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "238:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x8, #0, 243f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "str h13, [x25, #0x0]\n"
       "str h17, [x24, #0x0]\n"
       "str h21, [x23, #0x0]\n"
@@ -4289,19 +4289,19 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "239:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x8, #2, 241f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "tbz x8, #1, 240f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "st1 { v12.s }[2], [x25], #0x4\n"
       "st1 { v16.s }[2], [x24], #0x4\n"
       "st1 { v20.s }[2], [x23], #0x4\n"
       "st1 { v24.s }[2], [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "st1 { v12.h }[6], [x25]\n"
       "st1 { v16.h }[6], [x24]\n"
       "st1 { v20.h }[6], [x23]\n"
@@ -4309,7 +4309,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "240:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x8, #0, 243f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "st1 { v12.h }[4], [x25]\n"
       "st1 { v16.h }[4], [x24]\n"
       "st1 { v20.h }[4], [x23]\n"
@@ -4317,20 +4317,20 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 243f\n"
       "241:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x8, #1, 242f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "str s12, [x25], #0x4\n"
       "str s16, [x24], #0x4\n"
       "str s20, [x23], #0x4\n"
       "str s24, [x22], #0x4\n"
       "tbz x8, #0, 243f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "st1 { v12.h }[2], [x25]\n"
       "st1 { v16.h }[2], [x24]\n"
       "st1 { v20.h }[2], [x23]\n"
       "st1 { v24.h }[2], [x22]\n"
       "b 243f\n"
       "242:"  // Height 5: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "str h12, [x25, #0x0]\n"
       "str h16, [x24, #0x0]\n"
       "str h20, [x23, #0x0]\n"
@@ -4338,11 +4338,11 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "243:"  // Height 5: Partial direct writeback: Done
       "b 245f\n"
       "244:"  // Height 5: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -4364,25 +4364,25 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "bgt 198b\n"
       "b 296f\n"
       "246:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0xc\n"
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0xc\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "247:"  // Height 6: Column loop
-      "cbz x16, 248f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 248f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -4401,51 +4401,51 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 267f\n"
       "248:"  // Height 6: no bias
       "tbz %x[flags], #0, 266f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "cmp x8, #0x20\n"
-      "add x25, x15, x19, LSL #1\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "bge 265f\n"
       "tbz x8, #4, 256f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x15], #0x10\n"
-      "ld1 { v13.8h }, [x25], #0x10\n"
-      "ld1 { v17.8h }, [x24], #0x10\n"
       "ld1 { v20.8h }, [x23], #0x10\n"
       "ld1 { v24.8h }, [x22], #0x10\n"
       "ld1 { v28.8h }, [x21], #0x10\n"
+      "ld1 { v9.8h }, [x16], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
       "ld1 { v21.8h }, [x23], #0x10\n"
       "ld1 { v25.8h }, [x22], #0x10\n"
       "ld1 { v29.8h }, [x21], #0x10\n"
       "tbz x8, #3, 252f\n"
-      "ld1 { v10.8h }, [x15], #0x10\n"
+      "ld1 { v10.8h }, [x16], #0x10\n"
       "ld1 { v14.8h }, [x25], #0x10\n"
       "ld1 { v18.8h }, [x24], #0x10\n"
       "ld1 { v22.8h }, [x23], #0x10\n"
       "ld1 { v26.8h }, [x22], #0x10\n"
       "ld1 { v30.8h }, [x21], #0x10\n"
       "tbz x8, #2, 250f\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "ldr d27, [x22], #0x8\n"
       "ldr d31, [x21], #0x8\n"
       "tbz x8, #1, 249f\n"
-      "ld1 { v11.s }[2], [x15], #0x4\n"
-      "mov x19, #0x3c\n"
+      "ld1 { v11.s }[2], [x16], #0x4\n"
+      "mov x20, #0x3c\n"
       "ld1 { v15.s }[2], [x25], #0x4\n"
       "ld1 { v19.s }[2], [x24], #0x4\n"
       "ld1 { v23.s }[2], [x23], #0x4\n"
       "ld1 { v27.s }[2], [x22], #0x4\n"
       "ld1 { v31.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v11.h }[6], [x15]\n"
+      "ld1 { v11.h }[6], [x16]\n"
       "ld1 { v15.h }[6], [x25]\n"
       "ld1 { v19.h }[6], [x24]\n"
       "ld1 { v23.h }[6], [x23]\n"
@@ -4453,9 +4453,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v31.h }[6], [x21]\n"
       "b 264f\n"
       "249:"  // Height 6: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v11.h }[4], [x15]\n"
+      "ld1 { v11.h }[4], [x16]\n"
       "ld1 { v15.h }[4], [x25]\n"
       "ld1 { v19.h }[4], [x24]\n"
       "ld1 { v23.h }[4], [x23]\n"
@@ -4464,15 +4464,15 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "250:"  // Height 6: Partial accumulate: partial_2_24
       "tbz x8, #1, 251f\n"
-      "ldr s11, [x15], #0x4\n"
+      "ldr s11, [x16], #0x4\n"
+      "mov x20, #0x34\n"
       "ldr s15, [x25], #0x4\n"
-      "mov x19, #0x34\n"
       "ldr s19, [x24], #0x4\n"
       "ldr s23, [x23], #0x4\n"
       "ldr s27, [x22], #0x4\n"
       "ldr s31, [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v11.h }[2], [x15]\n"
+      "ld1 { v11.h }[2], [x16]\n"
       "ld1 { v15.h }[2], [x25]\n"
       "ld1 { v19.h }[2], [x24]\n"
       "ld1 { v23.h }[2], [x23]\n"
@@ -4480,9 +4480,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v31.h }[2], [x21]\n"
       "b 264f\n"
       "251:"  // Height 6: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 264f\n"
-      "ldr h11, [x15, #0x0]\n"
+      "ldr h11, [x16, #0x0]\n"
       "ldr h15, [x25, #0x0]\n"
       "ldr h19, [x24, #0x0]\n"
       "ldr h23, [x23, #0x0]\n"
@@ -4491,22 +4491,22 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "252:"  // Height 6: Partial accumulate: partial_4_16
       "tbz x8, #2, 254f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
       "ldr d14, [x25], #0x8\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "ldr d26, [x22], #0x8\n"
       "ldr d30, [x21], #0x8\n"
       "tbz x8, #1, 253f\n"
-      "ld1 { v10.s }[2], [x15], #0x4\n"
-      "mov x19, #0x2c\n"
+      "ld1 { v10.s }[2], [x16], #0x4\n"
+      "mov x20, #0x2c\n"
       "ld1 { v14.s }[2], [x25], #0x4\n"
       "ld1 { v18.s }[2], [x24], #0x4\n"
       "ld1 { v22.s }[2], [x23], #0x4\n"
       "ld1 { v26.s }[2], [x22], #0x4\n"
       "ld1 { v30.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v10.h }[6], [x15]\n"
+      "ld1 { v10.h }[6], [x16]\n"
       "ld1 { v14.h }[6], [x25]\n"
       "ld1 { v18.h }[6], [x24]\n"
       "ld1 { v22.h }[6], [x23]\n"
@@ -4514,9 +4514,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v30.h }[6], [x21]\n"
       "b 264f\n"
       "253:"  // Height 6: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v10.h }[4], [x15]\n"
+      "ld1 { v10.h }[4], [x16]\n"
       "ld1 { v14.h }[4], [x25]\n"
       "ld1 { v18.h }[4], [x24]\n"
       "ld1 { v22.h }[4], [x23]\n"
@@ -4525,15 +4525,15 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "254:"  // Height 6: Partial accumulate: partial_2_16
       "tbz x8, #1, 255f\n"
-      "ldr s10, [x15], #0x4\n"
+      "ldr s10, [x16], #0x4\n"
+      "mov x20, #0x24\n"
       "ldr s14, [x25], #0x4\n"
-      "mov x19, #0x24\n"
       "ldr s18, [x24], #0x4\n"
       "ldr s22, [x23], #0x4\n"
       "ldr s26, [x22], #0x4\n"
       "ldr s30, [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v10.h }[2], [x15]\n"
+      "ld1 { v10.h }[2], [x16]\n"
       "ld1 { v14.h }[2], [x25]\n"
       "ld1 { v18.h }[2], [x24]\n"
       "ld1 { v22.h }[2], [x23]\n"
@@ -4541,9 +4541,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v30.h }[2], [x21]\n"
       "b 264f\n"
       "255:"  // Height 6: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 264f\n"
-      "ldr h10, [x15, #0x0]\n"
+      "ldr h10, [x16, #0x0]\n"
       "ldr h14, [x25, #0x0]\n"
       "ldr h18, [x24, #0x0]\n"
       "ldr h22, [x23, #0x0]\n"
@@ -4552,29 +4552,29 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "256:"  // Height 6: Partial accumulate: partial_8_0
       "tbz x8, #3, 260f\n"
-      "ld1 { v8.8h }, [x15], #0x10\n"
+      "ld1 { v8.8h }, [x16], #0x10\n"
       "ld1 { v12.8h }, [x25], #0x10\n"
       "ld1 { v16.8h }, [x24], #0x10\n"
       "ld1 { v20.8h }, [x23], #0x10\n"
       "ld1 { v24.8h }, [x22], #0x10\n"
       "ld1 { v28.8h }, [x21], #0x10\n"
       "tbz x8, #2, 258f\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "ldr d29, [x21], #0x8\n"
       "tbz x8, #1, 257f\n"
-      "ld1 { v9.s }[2], [x15], #0x4\n"
-      "mov x19, #0x1c\n"
+      "ld1 { v9.s }[2], [x16], #0x4\n"
+      "mov x20, #0x1c\n"
       "ld1 { v13.s }[2], [x25], #0x4\n"
       "ld1 { v17.s }[2], [x24], #0x4\n"
       "ld1 { v21.s }[2], [x23], #0x4\n"
       "ld1 { v25.s }[2], [x22], #0x4\n"
       "ld1 { v29.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v9.h }[6], [x15]\n"
+      "ld1 { v9.h }[6], [x16]\n"
       "ld1 { v13.h }[6], [x25]\n"
       "ld1 { v17.h }[6], [x24]\n"
       "ld1 { v21.h }[6], [x23]\n"
@@ -4582,9 +4582,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v29.h }[6], [x21]\n"
       "b 264f\n"
       "257:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v9.h }[4], [x15]\n"
+      "ld1 { v9.h }[4], [x16]\n"
       "ld1 { v13.h }[4], [x25]\n"
       "ld1 { v17.h }[4], [x24]\n"
       "ld1 { v21.h }[4], [x23]\n"
@@ -4593,15 +4593,15 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "258:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x8, #1, 259f\n"
-      "ldr s9, [x15], #0x4\n"
+      "ldr s9, [x16], #0x4\n"
+      "mov x20, #0x14\n"
       "ldr s13, [x25], #0x4\n"
-      "mov x19, #0x14\n"
       "ldr s17, [x24], #0x4\n"
       "ldr s21, [x23], #0x4\n"
       "ldr s25, [x22], #0x4\n"
       "ldr s29, [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v9.h }[2], [x15]\n"
+      "ld1 { v9.h }[2], [x16]\n"
       "ld1 { v13.h }[2], [x25]\n"
       "ld1 { v17.h }[2], [x24]\n"
       "ld1 { v21.h }[2], [x23]\n"
@@ -4609,9 +4609,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v29.h }[2], [x21]\n"
       "b 264f\n"
       "259:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 264f\n"
-      "ldr h9, [x15, #0x0]\n"
+      "ldr h9, [x16, #0x0]\n"
       "ldr h13, [x25, #0x0]\n"
       "ldr h17, [x24, #0x0]\n"
       "ldr h21, [x23, #0x0]\n"
@@ -4620,22 +4620,22 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "260:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x8, #2, 262f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
       "ldr d12, [x25], #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "ldr d24, [x22], #0x8\n"
       "ldr d28, [x21], #0x8\n"
       "tbz x8, #1, 261f\n"
-      "ld1 { v8.s }[2], [x15], #0x4\n"
-      "mov x19, #0xc\n"
+      "ld1 { v8.s }[2], [x16], #0x4\n"
+      "mov x20, #0xc\n"
       "ld1 { v12.s }[2], [x25], #0x4\n"
       "ld1 { v16.s }[2], [x24], #0x4\n"
       "ld1 { v20.s }[2], [x23], #0x4\n"
       "ld1 { v24.s }[2], [x22], #0x4\n"
       "ld1 { v28.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v8.h }[6], [x15]\n"
+      "ld1 { v8.h }[6], [x16]\n"
       "ld1 { v12.h }[6], [x25]\n"
       "ld1 { v16.h }[6], [x24]\n"
       "ld1 { v20.h }[6], [x23]\n"
@@ -4643,9 +4643,9 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v28.h }[6], [x21]\n"
       "b 264f\n"
       "261:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v8.h }[4], [x15]\n"
+      "ld1 { v8.h }[4], [x16]\n"
       "ld1 { v12.h }[4], [x25]\n"
       "ld1 { v16.h }[4], [x24]\n"
       "ld1 { v20.h }[4], [x23]\n"
@@ -4654,15 +4654,15 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 264f\n"
       "262:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x8, #1, 263f\n"
-      "ldr s8, [x15], #0x4\n"
+      "ldr s8, [x16], #0x4\n"
+      "mov x20, #0x4\n"
       "ldr s12, [x25], #0x4\n"
-      "mov x19, #0x4\n"
       "ldr s16, [x24], #0x4\n"
       "ldr s20, [x23], #0x4\n"
       "ldr s24, [x22], #0x4\n"
       "ldr s28, [x21], #0x4\n"
       "tbz x8, #0, 264f\n"
-      "ld1 { v8.h }[2], [x15]\n"
+      "ld1 { v8.h }[2], [x16]\n"
       "ld1 { v12.h }[2], [x25]\n"
       "ld1 { v16.h }[2], [x24]\n"
       "ld1 { v20.h }[2], [x23]\n"
@@ -4670,21 +4670,21 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "ld1 { v28.h }[2], [x21]\n"
       "b 264f\n"
       "263:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr h8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr h12, [x25, #0x0]\n"
       "ldr h16, [x24, #0x0]\n"
       "ldr h20, [x23, #0x0]\n"
       "ldr h24, [x22, #0x0]\n"
       "ldr h28, [x21, #0x0]\n"
       "264:"  // Height 6: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 267f\n"
       "265:"  // Height 6: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -4732,404 +4732,404 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "267:"  // Height 6: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "268:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 269f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x14, 270f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #1\n"
-      "add x28, x28, x19, LSL #1\n"
-      "add x26, x26, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x15, 270f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #1\n"
+      "add x9, x9, x20, LSL #1\n"
+      "add x27, x27, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 270f\n"
       "269:"  // Height 6: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #1\n"
-      "add x26, x28, x19, LSL #1\n"
-      "add x24, x26, x19, LSL #1\n"
-      "add x22, x24, x19, LSL #1\n"
-      "add x20, x22, x19, LSL #1\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #1\n"
+      "add x27, x9, x20, LSL #1\n"
+      "add x25, x27, x20, LSL #1\n"
+      "add x23, x25, x20, LSL #1\n"
+      "add x21, x23, x20, LSL #1\n"
       "270:"  // Height 6: input setup done
-      "cmp x13, #0x8\n"
+      "cmp x14, #0x8\n"
       "blt 273f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x10\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x10\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 272f\n"
       "271:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr d6, [x17, #0x20]\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "ldr x10, [x17, #0x48]\n"
+      "add x21, x21, #0x10\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x28, x28, #0x10\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "add x26, x26, #0x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x21, #0x8]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
-      "add x24, x24, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x8\n"
       "fmla v20.8h, v6.8h, v3.h[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "cmp x14, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v28.8h, v6.8h, v5.h[1]\n"
       "ldr d6, [x17, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
-      "add x22, x22, #0x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v25.8h, v7.8h, v4.h[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v29.8h, v7.8h, v5.h[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
-      "add x20, x20, #0x10\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.8h, v6.8h, v2.h[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[1]\n"
-      "ldr x19, [x20, #0x8]\n"
       "fmla v30.8h, v6.8h, v5.h[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
-      "sub x13, x13, #0x8\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[1]\n"
-      "cmp x13, #0x10\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.8h, v7.8h, v2.h[1]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[1]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v27.8h, v7.8h, v4.h[1]\n"
       "fmla v31.8h, v7.8h, v5.h[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[2]\n"
       "fmla v12.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v24.8h, v6.8h, v4.h[2]\n"
       "fmla v28.8h, v6.8h, v5.h[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.8h, v7.8h, v0.h[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v25.8h, v7.8h, v4.h[2]\n"
       "fmla v29.8h, v7.8h, v5.h[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[2]\n"
       "fmla v14.8h, v6.8h, v1.h[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.8h, v6.8h, v2.h[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v26.8h, v6.8h, v4.h[2]\n"
       "fmla v30.8h, v6.8h, v5.h[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.8h, v7.8h, v0.h[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.8h, v7.8h, v2.h[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[2]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v27.8h, v7.8h, v4.h[2]\n"
       "fmla v31.8h, v7.8h, v5.h[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[3]\n"
       "fmla v12.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v24.8h, v6.8h, v4.h[3]\n"
       "fmla v28.8h, v6.8h, v5.h[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.8h, v7.8h, v0.h[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x108]\n"
       "fmla v17.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[3]\n"
-      "ldr x10, [x17, #0x108]\n"
       "fmla v25.8h, v7.8h, v4.h[3]\n"
       "fmla v29.8h, v7.8h, v5.h[3]\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[3]\n"
       "fmla v14.8h, v6.8h, v1.h[3]\n"
+      "ldr x11, [x17, #0x118]\n"
       "fmla v18.8h, v6.8h, v2.h[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[3]\n"
-      "ldr x11, [x17, #0x118]\n"
       "fmla v26.8h, v6.8h, v4.h[3]\n"
       "fmla v30.8h, v6.8h, v5.h[3]\n"
       "ldr d6, [x17, #0x100]\n"
       "fmla v11.8h, v7.8h, v0.h[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[3]\n"
+      "ldr x12, [x17, #0x128]\n"
       "fmla v19.8h, v7.8h, v2.h[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[3]\n"
-      "ldr x10, [x17, #0x128]\n"
       "fmla v27.8h, v7.8h, v4.h[3]\n"
       "fmla v31.8h, v7.8h, v5.h[3]\n"
       "ldr d7, [x17, #0x110]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[4]\n"
       "fmla v12.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x138]\n"
       "fmla v16.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[4]\n"
-      "ldr x11, [x17, #0x138]\n"
       "fmla v24.8h, v6.8h, v4.h[4]\n"
       "fmla v28.8h, v6.8h, v5.h[4]\n"
       "ldr d6, [x17, #0x120]\n"
       "fmla v9.8h, v7.8h, v0.h[4]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x148]\n"
       "fmla v17.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[4]\n"
-      "ldr x10, [x17, #0x148]\n"
       "fmla v25.8h, v7.8h, v4.h[4]\n"
       "fmla v29.8h, v7.8h, v5.h[4]\n"
       "ldr d7, [x17, #0x130]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[4]\n"
       "fmla v14.8h, v6.8h, v1.h[4]\n"
+      "ldr x11, [x17, #0x158]\n"
       "fmla v18.8h, v6.8h, v2.h[4]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[4]\n"
-      "ldr x11, [x17, #0x158]\n"
       "fmla v26.8h, v6.8h, v4.h[4]\n"
       "fmla v30.8h, v6.8h, v5.h[4]\n"
       "ldr d6, [x17, #0x140]\n"
       "fmla v11.8h, v7.8h, v0.h[4]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[4]\n"
+      "ldr x12, [x17, #0x168]\n"
       "fmla v19.8h, v7.8h, v2.h[4]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[4]\n"
-      "ldr x10, [x17, #0x168]\n"
       "fmla v27.8h, v7.8h, v4.h[4]\n"
       "fmla v31.8h, v7.8h, v5.h[4]\n"
       "ldr d7, [x17, #0x150]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[5]\n"
       "fmla v12.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x178]\n"
       "fmla v16.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[5]\n"
-      "ldr x11, [x17, #0x178]\n"
       "fmla v24.8h, v6.8h, v4.h[5]\n"
       "fmla v28.8h, v6.8h, v5.h[5]\n"
       "ldr d6, [x17, #0x160]\n"
       "fmla v9.8h, v7.8h, v0.h[5]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x188]\n"
       "fmla v17.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[5]\n"
-      "ldr x10, [x17, #0x188]\n"
       "fmla v25.8h, v7.8h, v4.h[5]\n"
       "fmla v29.8h, v7.8h, v5.h[5]\n"
       "ldr d7, [x17, #0x170]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[5]\n"
       "fmla v14.8h, v6.8h, v1.h[5]\n"
+      "ldr x11, [x17, #0x198]\n"
       "fmla v18.8h, v6.8h, v2.h[5]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[5]\n"
-      "ldr x11, [x17, #0x198]\n"
       "fmla v26.8h, v6.8h, v4.h[5]\n"
       "fmla v30.8h, v6.8h, v5.h[5]\n"
       "ldr d6, [x17, #0x180]\n"
       "fmla v11.8h, v7.8h, v0.h[5]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[5]\n"
+      "ldr x12, [x17, #0x1a8]\n"
       "fmla v19.8h, v7.8h, v2.h[5]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[5]\n"
-      "ldr x10, [x17, #0x1a8]\n"
       "fmla v27.8h, v7.8h, v4.h[5]\n"
       "fmla v31.8h, v7.8h, v5.h[5]\n"
       "ldr d7, [x17, #0x190]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[6]\n"
       "fmla v12.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1b8]\n"
       "fmla v16.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[6]\n"
-      "ldr x11, [x17, #0x1b8]\n"
       "fmla v24.8h, v6.8h, v4.h[6]\n"
       "fmla v28.8h, v6.8h, v5.h[6]\n"
       "ldr d6, [x17, #0x1a0]\n"
       "fmla v9.8h, v7.8h, v0.h[6]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1c8]\n"
       "fmla v17.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[6]\n"
-      "ldr x10, [x17, #0x1c8]\n"
       "fmla v25.8h, v7.8h, v4.h[6]\n"
       "fmla v29.8h, v7.8h, v5.h[6]\n"
       "ldr d7, [x17, #0x1b0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.8h, v6.8h, v0.h[6]\n"
       "fmla v14.8h, v6.8h, v1.h[6]\n"
+      "ldr x11, [x17, #0x1d8]\n"
       "fmla v18.8h, v6.8h, v2.h[6]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[6]\n"
-      "ldr x11, [x17, #0x1d8]\n"
       "fmla v26.8h, v6.8h, v4.h[6]\n"
       "fmla v30.8h, v6.8h, v5.h[6]\n"
       "ldr d6, [x17, #0x1c0]\n"
       "fmla v11.8h, v7.8h, v0.h[6]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.8h, v7.8h, v1.h[6]\n"
+      "ldr x12, [x17, #0x1e8]\n"
       "fmla v19.8h, v7.8h, v2.h[6]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.8h, v7.8h, v3.h[6]\n"
-      "ldr x10, [x17, #0x1e8]\n"
       "fmla v27.8h, v7.8h, v4.h[6]\n"
       "fmla v31.8h, v7.8h, v5.h[6]\n"
       "ldr d7, [x17, #0x1d0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.8h, v6.8h, v0.h[7]\n"
       "fmla v12.8h, v6.8h, v1.h[7]\n"
+      "ldr x11, [x17, #0x1f8]\n"
       "fmla v16.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.8h, v6.8h, v3.h[7]\n"
-      "ldr x11, [x17, #0x1f8]\n"
       "fmla v24.8h, v6.8h, v4.h[7]\n"
       "fmla v28.8h, v6.8h, v5.h[7]\n"
       "ldr d6, [x17, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.8h, v7.8h, v1.h[7]\n"
       "fmla v17.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.8h, v7.8h, v3.h[7]\n"
       "fmla v25.8h, v7.8h, v4.h[7]\n"
       "fmla v29.8h, v7.8h, v5.h[7]\n"
       "ldr d7, [x17, #0x1f0]\n"
-      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x200\n"
+      "fmla v10.8h, v6.8h, v0.h[7]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.8h, v6.8h, v1.h[7]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.8h, v6.8h, v2.h[7]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "fmla v26.8h, v6.8h, v4.h[7]\n"
       "fmla v30.8h, v6.8h, v5.h[7]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "mov v1.d[1], x27\n"
+      "ldr d4, [x23, #0x0]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "ldr d5, [x20, #0x0]\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
-      "mov v5.d[1], x19\n"
+      "ldr d5, [x21, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v5.d[1], x20\n"
+      "mov v7.d[1], x11\n"
       "bge 271b\n"
       "272:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "sub x13, x13, #0x8\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x17, #0x20]\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
+      "add x21, x21, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "add x26, x26, #0x10\n"
+      "sub x14, x14, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
-      "add x20, x20, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
       "ldr q6, [x17, #0x40]\n"
@@ -5336,18 +5336,18 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
       "273:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x13, 275f\n"
+      "cbz x14, 275f\n"
       "274:"  // Height 6: Multiply loop: Odd block loop
-      "ldr h0, [x12], #0x2\n"
-      "sub x13, x13, #0x1\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
+      "ldr h0, [x13], #0x2\n"
+      "sub x14, x14, #0x1\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
@@ -5374,29 +5374,27 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
-      "cbnz x13, 274b\n"
+      "cbnz x14, 274b\n"
       "275:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 268b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #1\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #1\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #1\n"
       "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 276f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.8h }, [x20]\n"
-      "ld1r { v0.8h }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.8h }, [x20]\n"
       "fmin v8.8h, v8.8h, v0.8h\n"
       "fmin v9.8h, v9.8h, v0.8h\n"
       "fmin v10.8h, v10.8h, v0.8h\n"
@@ -5407,16 +5405,6 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v15.8h, v15.8h, v0.8h\n"
       "fmin v16.8h, v16.8h, v0.8h\n"
       "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
       "fmin v18.8h, v18.8h, v0.8h\n"
       "fmin v19.8h, v19.8h, v0.8h\n"
       "fmin v20.8h, v20.8h, v0.8h\n"
@@ -5427,30 +5415,42 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "fmin v25.8h, v25.8h, v0.8h\n"
       "fmin v26.8h, v26.8h, v0.8h\n"
       "fmin v27.8h, v27.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
-      "fmax v24.8h, v24.8h, v1.8h\n"
-      "fmax v25.8h, v25.8h, v1.8h\n"
-      "fmax v26.8h, v26.8h, v1.8h\n"
-      "fmax v27.8h, v27.8h, v1.8h\n"
       "fmin v28.8h, v28.8h, v0.8h\n"
       "fmin v29.8h, v29.8h, v0.8h\n"
       "fmin v30.8h, v30.8h, v0.8h\n"
       "fmin v31.8h, v31.8h, v0.8h\n"
-      "fmax v28.8h, v28.8h, v1.8h\n"
-      "fmax v29.8h, v29.8h, v1.8h\n"
-      "fmax v30.8h, v30.8h, v1.8h\n"
-      "fmax v31.8h, v31.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
+      "fmax v24.8h, v24.8h, v0.8h\n"
+      "fmax v25.8h, v25.8h, v0.8h\n"
+      "fmax v26.8h, v26.8h, v0.8h\n"
+      "fmax v27.8h, v27.8h, v0.8h\n"
+      "fmax v28.8h, v28.8h, v0.8h\n"
+      "fmax v29.8h, v29.8h, v0.8h\n"
+      "fmax v30.8h, v30.8h, v0.8h\n"
+      "fmax v31.8h, v31.8h, v0.8h\n"
       "276:"  // Height 6: No activation
       "cmp x8, #0x20\n"
       "bge 293f\n"
       "tbz x8, #4, 284f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
-      "st1 { v9.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
+      "st1 { v9.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v13.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
@@ -5462,28 +5462,28 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "st1 { v28.8h }, [x21], #0x10\n"
       "st1 { v29.8h }, [x21], #0x10\n"
       "tbz x8, #3, 280f\n"
-      "st1 { v10.8h }, [x15], #0x10\n"
+      "st1 { v10.8h }, [x16], #0x10\n"
       "st1 { v14.8h }, [x25], #0x10\n"
       "st1 { v18.8h }, [x24], #0x10\n"
       "st1 { v22.8h }, [x23], #0x10\n"
       "st1 { v26.8h }, [x22], #0x10\n"
       "st1 { v30.8h }, [x21], #0x10\n"
       "tbz x8, #2, 278f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "str d27, [x22], #0x8\n"
       "str d31, [x21], #0x8\n"
       "tbz x8, #1, 277f\n"
-      "st1 { v11.s }[2], [x15], #0x4\n"
+      "st1 { v11.s }[2], [x16], #0x4\n"
       "st1 { v15.s }[2], [x25], #0x4\n"
       "st1 { v19.s }[2], [x24], #0x4\n"
       "st1 { v23.s }[2], [x23], #0x4\n"
       "st1 { v27.s }[2], [x22], #0x4\n"
       "st1 { v31.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v11.h }[6], [x15]\n"
+      "st1 { v11.h }[6], [x16]\n"
       "st1 { v15.h }[6], [x25]\n"
       "st1 { v19.h }[6], [x24]\n"
       "st1 { v23.h }[6], [x23]\n"
@@ -5492,7 +5492,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "277:"  // Height 6: Partial direct writeback: partial_1_28
       "tbz x8, #0, 292f\n"
-      "st1 { v11.h }[4], [x15]\n"
+      "st1 { v11.h }[4], [x16]\n"
       "st1 { v15.h }[4], [x25]\n"
       "st1 { v19.h }[4], [x24]\n"
       "st1 { v23.h }[4], [x23]\n"
@@ -5501,14 +5501,14 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "278:"  // Height 6: Partial direct writeback: partial_2_24
       "tbz x8, #1, 279f\n"
-      "str s11, [x15], #0x4\n"
+      "str s11, [x16], #0x4\n"
       "str s15, [x25], #0x4\n"
       "str s19, [x24], #0x4\n"
       "str s23, [x23], #0x4\n"
       "str s27, [x22], #0x4\n"
       "str s31, [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v11.h }[2], [x15]\n"
+      "st1 { v11.h }[2], [x16]\n"
       "st1 { v15.h }[2], [x25]\n"
       "st1 { v19.h }[2], [x24]\n"
       "st1 { v23.h }[2], [x23]\n"
@@ -5517,7 +5517,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "279:"  // Height 6: Partial direct writeback: partial_1_24
       "tbz x8, #0, 292f\n"
-      "str h11, [x15, #0x0]\n"
+      "str h11, [x16, #0x0]\n"
       "str h15, [x25, #0x0]\n"
       "str h19, [x24, #0x0]\n"
       "str h23, [x23, #0x0]\n"
@@ -5526,21 +5526,21 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "280:"  // Height 6: Partial direct writeback: partial_4_16
       "tbz x8, #2, 282f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "str d26, [x22], #0x8\n"
       "str d30, [x21], #0x8\n"
       "tbz x8, #1, 281f\n"
-      "st1 { v10.s }[2], [x15], #0x4\n"
+      "st1 { v10.s }[2], [x16], #0x4\n"
       "st1 { v14.s }[2], [x25], #0x4\n"
       "st1 { v18.s }[2], [x24], #0x4\n"
       "st1 { v22.s }[2], [x23], #0x4\n"
       "st1 { v26.s }[2], [x22], #0x4\n"
       "st1 { v30.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v10.h }[6], [x15]\n"
+      "st1 { v10.h }[6], [x16]\n"
       "st1 { v14.h }[6], [x25]\n"
       "st1 { v18.h }[6], [x24]\n"
       "st1 { v22.h }[6], [x23]\n"
@@ -5549,7 +5549,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "281:"  // Height 6: Partial direct writeback: partial_1_20
       "tbz x8, #0, 292f\n"
-      "st1 { v10.h }[4], [x15]\n"
+      "st1 { v10.h }[4], [x16]\n"
       "st1 { v14.h }[4], [x25]\n"
       "st1 { v18.h }[4], [x24]\n"
       "st1 { v22.h }[4], [x23]\n"
@@ -5558,14 +5558,14 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "282:"  // Height 6: Partial direct writeback: partial_2_16
       "tbz x8, #1, 283f\n"
-      "str s10, [x15], #0x4\n"
+      "str s10, [x16], #0x4\n"
       "str s14, [x25], #0x4\n"
       "str s18, [x24], #0x4\n"
       "str s22, [x23], #0x4\n"
       "str s26, [x22], #0x4\n"
       "str s30, [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v10.h }[2], [x15]\n"
+      "st1 { v10.h }[2], [x16]\n"
       "st1 { v14.h }[2], [x25]\n"
       "st1 { v18.h }[2], [x24]\n"
       "st1 { v22.h }[2], [x23]\n"
@@ -5574,7 +5574,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "283:"  // Height 6: Partial direct writeback: partial_1_16
       "tbz x8, #0, 292f\n"
-      "str h10, [x15, #0x0]\n"
+      "str h10, [x16, #0x0]\n"
       "str h14, [x25, #0x0]\n"
       "str h18, [x24, #0x0]\n"
       "str h22, [x23, #0x0]\n"
@@ -5583,28 +5583,28 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "284:"  // Height 6: Partial direct writeback: partial_8_0
       "tbz x8, #3, 288f\n"
-      "st1 { v8.8h }, [x15], #0x10\n"
+      "st1 { v8.8h }, [x16], #0x10\n"
       "st1 { v12.8h }, [x25], #0x10\n"
       "st1 { v16.8h }, [x24], #0x10\n"
       "st1 { v20.8h }, [x23], #0x10\n"
       "st1 { v24.8h }, [x22], #0x10\n"
       "st1 { v28.8h }, [x21], #0x10\n"
       "tbz x8, #2, 286f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "str d29, [x21], #0x8\n"
       "tbz x8, #1, 285f\n"
-      "st1 { v9.s }[2], [x15], #0x4\n"
+      "st1 { v9.s }[2], [x16], #0x4\n"
       "st1 { v13.s }[2], [x25], #0x4\n"
       "st1 { v17.s }[2], [x24], #0x4\n"
       "st1 { v21.s }[2], [x23], #0x4\n"
       "st1 { v25.s }[2], [x22], #0x4\n"
       "st1 { v29.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v9.h }[6], [x15]\n"
+      "st1 { v9.h }[6], [x16]\n"
       "st1 { v13.h }[6], [x25]\n"
       "st1 { v17.h }[6], [x24]\n"
       "st1 { v21.h }[6], [x23]\n"
@@ -5613,7 +5613,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "285:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x8, #0, 292f\n"
-      "st1 { v9.h }[4], [x15]\n"
+      "st1 { v9.h }[4], [x16]\n"
       "st1 { v13.h }[4], [x25]\n"
       "st1 { v17.h }[4], [x24]\n"
       "st1 { v21.h }[4], [x23]\n"
@@ -5622,14 +5622,14 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "286:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x8, #1, 287f\n"
-      "str s9, [x15], #0x4\n"
+      "str s9, [x16], #0x4\n"
       "str s13, [x25], #0x4\n"
       "str s17, [x24], #0x4\n"
       "str s21, [x23], #0x4\n"
       "str s25, [x22], #0x4\n"
       "str s29, [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v9.h }[2], [x15]\n"
+      "st1 { v9.h }[2], [x16]\n"
       "st1 { v13.h }[2], [x25]\n"
       "st1 { v17.h }[2], [x24]\n"
       "st1 { v21.h }[2], [x23]\n"
@@ -5638,7 +5638,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "287:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x8, #0, 292f\n"
-      "str h9, [x15, #0x0]\n"
+      "str h9, [x16, #0x0]\n"
       "str h13, [x25, #0x0]\n"
       "str h17, [x24, #0x0]\n"
       "str h21, [x23, #0x0]\n"
@@ -5647,21 +5647,21 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "288:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x8, #2, 290f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "str d28, [x21], #0x8\n"
       "tbz x8, #1, 289f\n"
-      "st1 { v8.s }[2], [x15], #0x4\n"
+      "st1 { v8.s }[2], [x16], #0x4\n"
       "st1 { v12.s }[2], [x25], #0x4\n"
       "st1 { v16.s }[2], [x24], #0x4\n"
       "st1 { v20.s }[2], [x23], #0x4\n"
       "st1 { v24.s }[2], [x22], #0x4\n"
       "st1 { v28.s }[2], [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v8.h }[6], [x15]\n"
+      "st1 { v8.h }[6], [x16]\n"
       "st1 { v12.h }[6], [x25]\n"
       "st1 { v16.h }[6], [x24]\n"
       "st1 { v20.h }[6], [x23]\n"
@@ -5670,7 +5670,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "289:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x8, #0, 292f\n"
-      "st1 { v8.h }[4], [x15]\n"
+      "st1 { v8.h }[4], [x16]\n"
       "st1 { v12.h }[4], [x25]\n"
       "st1 { v16.h }[4], [x24]\n"
       "st1 { v20.h }[4], [x23]\n"
@@ -5679,14 +5679,14 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "b 292f\n"
       "290:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x8, #1, 291f\n"
-      "str s8, [x15], #0x4\n"
+      "str s8, [x16], #0x4\n"
       "str s12, [x25], #0x4\n"
       "str s16, [x24], #0x4\n"
       "str s20, [x23], #0x4\n"
       "str s24, [x22], #0x4\n"
       "str s28, [x21], #0x4\n"
       "tbz x8, #0, 292f\n"
-      "st1 { v8.h }[2], [x15]\n"
+      "st1 { v8.h }[2], [x16]\n"
       "st1 { v12.h }[2], [x25]\n"
       "st1 { v16.h }[2], [x24]\n"
       "st1 { v20.h }[2], [x23]\n"
@@ -5694,7 +5694,7 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "st1 { v28.h }[2], [x21]\n"
       "b 292f\n"
       "291:"  // Height 6: Partial direct writeback: partial_1_0
-      "str h8, [x15, #0x0]\n"
+      "str h8, [x16, #0x0]\n"
       "str h12, [x25, #0x0]\n"
       "str h16, [x24, #0x0]\n"
       "str h20, [x23, #0x0]\n"
@@ -5703,11 +5703,11 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "292:"  // Height 6: Partial direct writeback: Done
       "b 294f\n"
       "293:"  // Height 6: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -5733,20 +5733,20 @@ void a64_hybrid_fp16_mla_6x32_a55 (
       "bgt 247b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 296f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 295f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "295:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "296:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp
index 8877306f40..335308751f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
 
@@ -101,138 +101,138 @@ void a64_hybrid_fp16_mla_6x32 (
       "cmp %x[M], #0x2\n"
       "bgt 99f\n"
       "beq 50f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x9, 3f\n"
-      "ldr q8, [x9, #0x0]\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "cbz x12, 3f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "b 22f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 21f\n"
       "cmp x11, #0x20\n"
       "bge 20f\n"
       "tbz x11, #4, 11f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
       "tbz x11, #3, 7f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
       "tbz x11, #2, 5f\n"
-      "ldr d11, [x28], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
       "tbz x11, #1, 4f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "mov x20, #0x3c\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v11.h }[6], [x28]\n"
+      "ld1 { v11.h }[6], [x9]\n"
       "b 19f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v11.h }[4], [x28]\n"
+      "ld1 { v11.h }[4], [x9]\n"
       "b 19f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_24
       "tbz x11, #1, 6f\n"
-      "ldr s11, [x28], #0x4\n"
-      "mov x19, #0x34\n"
+      "ldr s11, [x9], #0x4\n"
+      "mov x20, #0x34\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v11.h }[2], [x28]\n"
+      "ld1 { v11.h }[2], [x9]\n"
       "b 19f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 19f\n"
-      "ldr h11, [x28, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
       "b 19f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_16
       "tbz x11, #2, 9f\n"
-      "ldr d10, [x28], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
       "tbz x11, #1, 8f\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "mov x19, #0x2c\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "mov x20, #0x2c\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v10.h }[6], [x28]\n"
+      "ld1 { v10.h }[6], [x9]\n"
       "b 19f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v10.h }[4], [x28]\n"
+      "ld1 { v10.h }[4], [x9]\n"
       "b 19f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_16
       "tbz x11, #1, 10f\n"
-      "ldr s10, [x28], #0x4\n"
-      "mov x19, #0x24\n"
+      "ldr s10, [x9], #0x4\n"
+      "mov x20, #0x24\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v10.h }[2], [x28]\n"
+      "ld1 { v10.h }[2], [x9]\n"
       "b 19f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 19f\n"
-      "ldr h10, [x28, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
       "b 19f\n"
       "11:"  // Height 1: Partial accumulate: partial_8_0
       "tbz x11, #3, 15f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
       "tbz x11, #2, 13f\n"
-      "ldr d9, [x28], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
       "tbz x11, #1, 12f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "mov x20, #0x1c\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v9.h }[6], [x28]\n"
+      "ld1 { v9.h }[6], [x9]\n"
       "b 19f\n"
       "12:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v9.h }[4], [x28]\n"
+      "ld1 { v9.h }[4], [x9]\n"
       "b 19f\n"
       "13:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x11, #1, 14f\n"
-      "ldr s9, [x28], #0x4\n"
-      "mov x19, #0x14\n"
+      "ldr s9, [x9], #0x4\n"
+      "mov x20, #0x14\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v9.h }[2], [x28]\n"
+      "ld1 { v9.h }[2], [x9]\n"
       "b 19f\n"
       "14:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 19f\n"
-      "ldr h9, [x28, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
       "b 19f\n"
       "15:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x11, #2, 17f\n"
-      "ldr d8, [x28], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
       "tbz x11, #1, 16f\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "mov x19, #0xc\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "mov x20, #0xc\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v8.h }[6], [x28]\n"
+      "ld1 { v8.h }[6], [x9]\n"
       "b 19f\n"
       "16:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v8.h }[4], [x28]\n"
+      "ld1 { v8.h }[4], [x9]\n"
       "b 19f\n"
       "17:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x11, #1, 18f\n"
-      "ldr s8, [x28], #0x4\n"
-      "mov x19, #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "mov x20, #0x4\n"
       "tbz x11, #0, 19f\n"
-      "ld1 { v8.h }[2], [x28]\n"
+      "ld1 { v8.h }[2], [x9]\n"
       "b 19f\n"
       "18:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr h8, [x9, #0x0]\n"
+      "mov x20, #0x0\n"
       "19:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 22f\n"
       "20:"  // Height 1: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
       "b 22f\n"
       "21:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -240,46 +240,42 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "22:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "23:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 24f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 25f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 25f\n"
       "24:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "25:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 28f\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 27f\n"
       "26:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x8\n"
-      "fmla v10.8h, v6.8h, v0.h[0]\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x10\n"
-      "fmla v11.8h, v7.8h, v0.h[0]\n"
+      "fmla v10.8h, v6.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "fmla v11.8h, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[1]\n"
@@ -328,27 +324,28 @@ void a64_hybrid_fp16_mla_6x32 (
       "ldr q6, [x10, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "ldr q7, [x10, #0x1f0]\n"
-      "add x10, x10, #0x200\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
-      "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "add x10, x10, #0x200\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 26b\n"
       "27:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
-      "fmla v10.8h, v6.8h, v0.h[0]\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "fmla v10.8h, v6.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
@@ -399,325 +396,328 @@ void a64_hybrid_fp16_mla_6x32 (
       "ldr q6, [x10, #0x1e0]\n"
       "fmla v9.8h, v7.8h, v0.h[7]\n"
       "ldr q7, [x10, #0x1f0]\n"
-      "add x10, x10, #0x200\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x8\n"
       "fmla v10.8h, v6.8h, v0.h[7]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x200\n"
       "28:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 30f\n"
+      "cbz x27, 30f\n"
       "29:"  // Height 1: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
+      "ldr h0, [x26], #0x2\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
+      "sub x27, x27, #0x1\n"
       "ldr q7, [x10, #0x10]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
+      "ldr q7, [x10, #0x30]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
-      "cbnz x26, 29b\n"
+      "add x10, x10, #0x40\n"
+      "cbnz x27, 29b\n"
       "30:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 23b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "tbz %x[flags], #1, 31f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
       "31:"  // Height 1: No activation
       "cmp x11, #0x20\n"
       "bge 48f\n"
       "tbz x11, #4, 39f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
       "tbz x11, #3, 35f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
       "tbz x11, #2, 33f\n"
-      "str d11, [x28], #0x8\n"
+      "str d11, [x9], #0x8\n"
       "tbz x11, #1, 32f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v11.h }[6], [x28]\n"
+      "st1 { v11.h }[6], [x9]\n"
       "b 47f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_28
       "tbz x11, #0, 47f\n"
-      "st1 { v11.h }[4], [x28]\n"
+      "st1 { v11.h }[4], [x9]\n"
       "b 47f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_24
       "tbz x11, #1, 34f\n"
-      "str s11, [x28], #0x4\n"
+      "str s11, [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v11.h }[2], [x28]\n"
+      "st1 { v11.h }[2], [x9]\n"
       "b 47f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_24
       "tbz x11, #0, 47f\n"
-      "str h11, [x28, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
       "b 47f\n"
       "35:"  // Height 1: Partial direct writeback: partial_4_16
       "tbz x11, #2, 37f\n"
-      "str d10, [x28], #0x8\n"
+      "str d10, [x9], #0x8\n"
       "tbz x11, #1, 36f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v10.h }[6], [x28]\n"
+      "st1 { v10.h }[6], [x9]\n"
       "b 47f\n"
       "36:"  // Height 1: Partial direct writeback: partial_1_20
       "tbz x11, #0, 47f\n"
-      "st1 { v10.h }[4], [x28]\n"
+      "st1 { v10.h }[4], [x9]\n"
       "b 47f\n"
       "37:"  // Height 1: Partial direct writeback: partial_2_16
       "tbz x11, #1, 38f\n"
-      "str s10, [x28], #0x4\n"
+      "str s10, [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v10.h }[2], [x28]\n"
+      "st1 { v10.h }[2], [x9]\n"
       "b 47f\n"
       "38:"  // Height 1: Partial direct writeback: partial_1_16
       "tbz x11, #0, 47f\n"
-      "str h10, [x28, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
       "b 47f\n"
       "39:"  // Height 1: Partial direct writeback: partial_8_0
       "tbz x11, #3, 43f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
       "tbz x11, #2, 41f\n"
-      "str d9, [x28], #0x8\n"
+      "str d9, [x9], #0x8\n"
       "tbz x11, #1, 40f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v9.h }[6], [x28]\n"
+      "st1 { v9.h }[6], [x9]\n"
       "b 47f\n"
       "40:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x11, #0, 47f\n"
-      "st1 { v9.h }[4], [x28]\n"
+      "st1 { v9.h }[4], [x9]\n"
       "b 47f\n"
       "41:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x11, #1, 42f\n"
-      "str s9, [x28], #0x4\n"
+      "str s9, [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v9.h }[2], [x28]\n"
+      "st1 { v9.h }[2], [x9]\n"
       "b 47f\n"
       "42:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x11, #0, 47f\n"
-      "str h9, [x28, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
       "b 47f\n"
       "43:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x11, #2, 45f\n"
-      "str d8, [x28], #0x8\n"
+      "str d8, [x9], #0x8\n"
       "tbz x11, #1, 44f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v8.h }[6], [x28]\n"
+      "st1 { v8.h }[6], [x9]\n"
       "b 47f\n"
       "44:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x11, #0, 47f\n"
-      "st1 { v8.h }[4], [x28]\n"
+      "st1 { v8.h }[4], [x9]\n"
       "b 47f\n"
       "45:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x11, #1, 46f\n"
-      "str s8, [x28], #0x4\n"
+      "str s8, [x9], #0x4\n"
       "tbz x11, #0, 47f\n"
-      "st1 { v8.h }[2], [x28]\n"
+      "st1 { v8.h }[2], [x9]\n"
       "b 47f\n"
       "46:"  // Height 1: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
       "47:"  // Height 1: Partial direct writeback: Done
       "b 49f\n"
       "48:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "49:"  // Height 1: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 2b\n"
       "b 296f\n"
       "50:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "51:"  // Height 2: Column loop
-      "cbz x9, 52f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 52f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "add x12, x12, #0x40\n"
       "b 71f\n"
       "52:"  // Height 2: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x11, #0x20\n"
-      "add x24, x28, x19, LSL #1\n"
+      "add x25, x9, x20, LSL #1\n"
       "bge 69f\n"
       "tbz x11, #4, 60f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
       "tbz x11, #3, 56f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
       "tbz x11, #2, 54f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
       "tbz x11, #1, 53f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v11.h }[6], [x28]\n"
-      "ld1 { v15.h }[6], [x24]\n"
+      "ld1 { v11.h }[6], [x9]\n"
+      "ld1 { v15.h }[6], [x25]\n"
       "b 68f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v11.h }[4], [x28]\n"
-      "ld1 { v15.h }[4], [x24]\n"
+      "ld1 { v11.h }[4], [x9]\n"
+      "ld1 { v15.h }[4], [x25]\n"
       "b 68f\n"
       "54:"  // Height 2: Partial accumulate: partial_2_24
       "tbz x11, #1, 55f\n"
-      "ldr s11, [x28], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
+      "ldr s11, [x9], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v11.h }[2], [x28]\n"
-      "ld1 { v15.h }[2], [x24]\n"
+      "ld1 { v11.h }[2], [x9]\n"
+      "ld1 { v15.h }[2], [x25]\n"
       "b 68f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 68f\n"
-      "ldr h11, [x28, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
       "b 68f\n"
       "56:"  // Height 2: Partial accumulate: partial_4_16
       "tbz x11, #2, 58f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
       "tbz x11, #1, 57f\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v10.h }[6], [x28]\n"
-      "ld1 { v14.h }[6], [x24]\n"
+      "ld1 { v10.h }[6], [x9]\n"
+      "ld1 { v14.h }[6], [x25]\n"
       "b 68f\n"
       "57:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v10.h }[4], [x28]\n"
-      "ld1 { v14.h }[4], [x24]\n"
+      "ld1 { v10.h }[4], [x9]\n"
+      "ld1 { v14.h }[4], [x25]\n"
       "b 68f\n"
       "58:"  // Height 2: Partial accumulate: partial_2_16
       "tbz x11, #1, 59f\n"
-      "ldr s10, [x28], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
+      "ldr s10, [x9], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v10.h }[2], [x28]\n"
-      "ld1 { v14.h }[2], [x24]\n"
+      "ld1 { v10.h }[2], [x9]\n"
+      "ld1 { v14.h }[2], [x25]\n"
       "b 68f\n"
       "59:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 68f\n"
-      "ldr h10, [x28, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
       "b 68f\n"
       "60:"  // Height 2: Partial accumulate: partial_8_0
       "tbz x11, #3, 64f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
       "tbz x11, #2, 62f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
       "tbz x11, #1, 61f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v9.h }[6], [x28]\n"
-      "ld1 { v13.h }[6], [x24]\n"
+      "ld1 { v9.h }[6], [x9]\n"
+      "ld1 { v13.h }[6], [x25]\n"
       "b 68f\n"
       "61:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v9.h }[4], [x28]\n"
-      "ld1 { v13.h }[4], [x24]\n"
+      "ld1 { v9.h }[4], [x9]\n"
+      "ld1 { v13.h }[4], [x25]\n"
       "b 68f\n"
       "62:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x11, #1, 63f\n"
-      "ldr s9, [x28], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
+      "ldr s9, [x9], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v9.h }[2], [x28]\n"
-      "ld1 { v13.h }[2], [x24]\n"
+      "ld1 { v9.h }[2], [x9]\n"
+      "ld1 { v13.h }[2], [x25]\n"
       "b 68f\n"
       "63:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 68f\n"
-      "ldr h9, [x28, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
       "b 68f\n"
       "64:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x11, #2, 66f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
       "tbz x11, #1, 65f\n"
-      "mov x19, #0xc\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v8.h }[6], [x28]\n"
-      "ld1 { v12.h }[6], [x24]\n"
+      "ld1 { v8.h }[6], [x9]\n"
+      "ld1 { v12.h }[6], [x25]\n"
       "b 68f\n"
       "65:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v8.h }[4], [x28]\n"
-      "ld1 { v12.h }[4], [x24]\n"
+      "ld1 { v8.h }[4], [x9]\n"
+      "ld1 { v12.h }[4], [x25]\n"
       "b 68f\n"
       "66:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x11, #1, 67f\n"
-      "ldr s8, [x28], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
       "tbz x11, #0, 68f\n"
-      "ld1 { v8.h }[2], [x28]\n"
-      "ld1 { v12.h }[2], [x24]\n"
+      "ld1 { v8.h }[2], [x9]\n"
+      "ld1 { v12.h }[2], [x25]\n"
       "b 68f\n"
       "67:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h12, [x24, #0x0]\n"
+      "ldr h8, [x9, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "68:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 71f\n"
       "69:"  // Height 2: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "b 71f\n"
       "70:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -729,58 +729,58 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "71:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "72:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 74f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 74f\n"
       "73:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "74:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 77f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 76f\n"
       "75:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "add x25, x25, #0x10\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "cmp x27, #0x10\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[1]\n"
       "fmla v14.8h, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x80]\n"
@@ -858,32 +858,32 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v14.8h, v6.8h, v1.h[7]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 75b\n"
       "76:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x25, x25, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "sub x27, x27, #0x8\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v9.8h, v7.8h, v0.h[1]\n"
       "fmla v13.8h, v7.8h, v1.h[1]\n"
       "ldr q7, [x10, #0x70]\n"
@@ -965,414 +965,414 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v11.8h, v7.8h, v0.h[7]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "77:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 79f\n"
+      "cbz x27, 79f\n"
       "78:"  // Height 2: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h1, [x24], #0x2\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "add x10, x10, #0x40\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
-      "cbnz x26, 78b\n"
+      "cbnz x27, 78b\n"
       "79:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 80f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmin v12.8h, v12.8h, v0.8h\n"
-      "fmin v13.8h, v13.8h, v0.8h\n"
-      "fmin v14.8h, v14.8h, v0.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmin v12.8h, v12.8h, v1.8h\n"
+      "fmin v13.8h, v13.8h, v1.8h\n"
+      "fmin v14.8h, v14.8h, v1.8h\n"
+      "fmin v15.8h, v15.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
       "80:"  // Height 2: No activation
       "cmp x11, #0x20\n"
       "bge 97f\n"
       "tbz x11, #4, 88f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
       "tbz x11, #3, 84f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
       "tbz x11, #2, 82f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
       "tbz x11, #1, 81f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v11.h }[6], [x28]\n"
-      "st1 { v15.h }[6], [x24]\n"
+      "st1 { v11.h }[6], [x9]\n"
+      "st1 { v15.h }[6], [x25]\n"
       "b 96f\n"
       "81:"  // Height 2: Partial direct writeback: partial_1_28
       "tbz x11, #0, 96f\n"
-      "st1 { v11.h }[4], [x28]\n"
-      "st1 { v15.h }[4], [x24]\n"
+      "st1 { v11.h }[4], [x9]\n"
+      "st1 { v15.h }[4], [x25]\n"
       "b 96f\n"
       "82:"  // Height 2: Partial direct writeback: partial_2_24
       "tbz x11, #1, 83f\n"
-      "str s11, [x28], #0x4\n"
-      "str s15, [x24], #0x4\n"
+      "str s11, [x9], #0x4\n"
+      "str s15, [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v11.h }[2], [x28]\n"
-      "st1 { v15.h }[2], [x24]\n"
+      "st1 { v11.h }[2], [x9]\n"
+      "st1 { v15.h }[2], [x25]\n"
       "b 96f\n"
       "83:"  // Height 2: Partial direct writeback: partial_1_24
       "tbz x11, #0, 96f\n"
-      "str h11, [x28, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
       "b 96f\n"
       "84:"  // Height 2: Partial direct writeback: partial_4_16
       "tbz x11, #2, 86f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
       "tbz x11, #1, 85f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v10.h }[6], [x28]\n"
-      "st1 { v14.h }[6], [x24]\n"
+      "st1 { v10.h }[6], [x9]\n"
+      "st1 { v14.h }[6], [x25]\n"
       "b 96f\n"
       "85:"  // Height 2: Partial direct writeback: partial_1_20
       "tbz x11, #0, 96f\n"
-      "st1 { v10.h }[4], [x28]\n"
-      "st1 { v14.h }[4], [x24]\n"
+      "st1 { v10.h }[4], [x9]\n"
+      "st1 { v14.h }[4], [x25]\n"
       "b 96f\n"
       "86:"  // Height 2: Partial direct writeback: partial_2_16
       "tbz x11, #1, 87f\n"
-      "str s10, [x28], #0x4\n"
-      "str s14, [x24], #0x4\n"
+      "str s10, [x9], #0x4\n"
+      "str s14, [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v10.h }[2], [x28]\n"
-      "st1 { v14.h }[2], [x24]\n"
+      "st1 { v10.h }[2], [x9]\n"
+      "st1 { v14.h }[2], [x25]\n"
       "b 96f\n"
       "87:"  // Height 2: Partial direct writeback: partial_1_16
       "tbz x11, #0, 96f\n"
-      "str h10, [x28, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
       "b 96f\n"
       "88:"  // Height 2: Partial direct writeback: partial_8_0
       "tbz x11, #3, 92f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
       "tbz x11, #2, 90f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
       "tbz x11, #1, 89f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v9.h }[6], [x28]\n"
-      "st1 { v13.h }[6], [x24]\n"
+      "st1 { v9.h }[6], [x9]\n"
+      "st1 { v13.h }[6], [x25]\n"
       "b 96f\n"
       "89:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x11, #0, 96f\n"
-      "st1 { v9.h }[4], [x28]\n"
-      "st1 { v13.h }[4], [x24]\n"
+      "st1 { v9.h }[4], [x9]\n"
+      "st1 { v13.h }[4], [x25]\n"
       "b 96f\n"
       "90:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x11, #1, 91f\n"
-      "str s9, [x28], #0x4\n"
-      "str s13, [x24], #0x4\n"
+      "str s9, [x9], #0x4\n"
+      "str s13, [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v9.h }[2], [x28]\n"
-      "st1 { v13.h }[2], [x24]\n"
+      "st1 { v9.h }[2], [x9]\n"
+      "st1 { v13.h }[2], [x25]\n"
       "b 96f\n"
       "91:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x11, #0, 96f\n"
-      "str h9, [x28, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
       "b 96f\n"
       "92:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x11, #2, 94f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
       "tbz x11, #1, 93f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v8.h }[6], [x28]\n"
-      "st1 { v12.h }[6], [x24]\n"
+      "st1 { v8.h }[6], [x9]\n"
+      "st1 { v12.h }[6], [x25]\n"
       "b 96f\n"
       "93:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x11, #0, 96f\n"
-      "st1 { v8.h }[4], [x28]\n"
-      "st1 { v12.h }[4], [x24]\n"
+      "st1 { v8.h }[4], [x9]\n"
+      "st1 { v12.h }[4], [x25]\n"
       "b 96f\n"
       "94:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x11, #1, 95f\n"
-      "str s8, [x28], #0x4\n"
-      "str s12, [x24], #0x4\n"
+      "str s8, [x9], #0x4\n"
+      "str s12, [x25], #0x4\n"
       "tbz x11, #0, 96f\n"
-      "st1 { v8.h }[2], [x28]\n"
-      "st1 { v12.h }[2], [x24]\n"
+      "st1 { v8.h }[2], [x9]\n"
+      "st1 { v12.h }[2], [x25]\n"
       "b 96f\n"
       "95:"  // Height 2: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
       "96:"  // Height 2: Partial direct writeback: Done
       "b 98f\n"
       "97:"  // Height 2: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
       "98:"  // Height 2: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 51b\n"
       "b 296f\n"
       "99:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "100:"  // Height 3: Column loop
-      "cbz x9, 101f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 101f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
       "mov v13.16b, v9.16b\n"
-      "add x9, x9, #0x40\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 120f\n"
       "101:"  // Height 3: no bias
       "tbz %x[flags], #0, 119f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
       "cmp x11, #0x20\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "bge 118f\n"
       "tbz x11, #4, 109f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
       "tbz x11, #3, 105f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
       "tbz x11, #2, 103f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
       "tbz x11, #1, 102f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v11.h }[6], [x28]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
+      "ld1 { v11.h }[6], [x9]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
       "b 117f\n"
       "102:"  // Height 3: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v11.h }[4], [x28]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
+      "ld1 { v11.h }[4], [x9]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
       "b 117f\n"
       "103:"  // Height 3: Partial accumulate: partial_2_24
       "tbz x11, #1, 104f\n"
-      "ldr s11, [x28], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
+      "ldr s11, [x9], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v11.h }[2], [x28]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
+      "ld1 { v11.h }[2], [x9]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
       "b 117f\n"
       "104:"  // Height 3: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 117f\n"
-      "ldr h11, [x28, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
       "b 117f\n"
       "105:"  // Height 3: Partial accumulate: partial_4_16
       "tbz x11, #2, 107f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
       "tbz x11, #1, 106f\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v10.h }[6], [x28]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
+      "ld1 { v10.h }[6], [x9]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
       "b 117f\n"
       "106:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v10.h }[4], [x28]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
+      "ld1 { v10.h }[4], [x9]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
       "b 117f\n"
       "107:"  // Height 3: Partial accumulate: partial_2_16
       "tbz x11, #1, 108f\n"
-      "ldr s10, [x28], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
+      "ldr s10, [x9], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v10.h }[2], [x28]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
+      "ld1 { v10.h }[2], [x9]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
       "b 117f\n"
       "108:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 117f\n"
-      "ldr h10, [x28, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
       "b 117f\n"
       "109:"  // Height 3: Partial accumulate: partial_8_0
       "tbz x11, #3, 113f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
       "tbz x11, #2, 111f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
       "tbz x11, #1, 110f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v9.h }[6], [x28]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
+      "ld1 { v9.h }[6], [x9]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
       "b 117f\n"
       "110:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v9.h }[4], [x28]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
+      "ld1 { v9.h }[4], [x9]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
       "b 117f\n"
       "111:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x11, #1, 112f\n"
-      "ldr s9, [x28], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
+      "ldr s9, [x9], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v9.h }[2], [x28]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
+      "ld1 { v9.h }[2], [x9]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
       "b 117f\n"
       "112:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 117f\n"
-      "ldr h9, [x28, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
       "b 117f\n"
       "113:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x11, #2, 115f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
       "tbz x11, #1, 114f\n"
-      "mov x19, #0xc\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v8.h }[6], [x28]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
+      "ld1 { v8.h }[6], [x9]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
       "b 117f\n"
       "114:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v8.h }[4], [x28]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
+      "ld1 { v8.h }[4], [x9]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
       "b 117f\n"
       "115:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x11, #1, 116f\n"
-      "ldr s8, [x28], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
       "tbz x11, #0, 117f\n"
-      "ld1 { v8.h }[2], [x28]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
+      "ld1 { v8.h }[2], [x9]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
       "b 117f\n"
       "116:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h12, [x24, #0x0]\n"
-      "ldr h16, [x23, #0x0]\n"
+      "ldr h8, [x9, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
       "117:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 120f\n"
       "118:"  // Height 3: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
       "b 120f\n"
       "119:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1388,62 +1388,62 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "120:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "121:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 122f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 123f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 123f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 123f\n"
       "122:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "123:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 126f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 125f\n"
       "124:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x8\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x24, x24, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "cmp x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v8.8h, v6.8h, v0.h[1]\n"
       "fmla v12.8h, v6.8h, v1.h[1]\n"
       "fmla v16.8h, v6.8h, v2.h[1]\n"
@@ -1554,34 +1554,34 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v18.8h, v6.8h, v2.h[7]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 124b\n"
       "125:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "ldr q7, [x10, #0x30]\n"
+      "sub x27, x27, #0x8\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -1697,16 +1697,16 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v15.8h, v7.8h, v1.h[7]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "126:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 128f\n"
+      "cbz x27, 128f\n"
       "127:"  // Height 3: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
@@ -1720,483 +1720,483 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
-      "cbnz x26, 127b\n"
+      "cbnz x27, 127b\n"
       "128:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 121b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 129f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmin v12.8h, v12.8h, v0.8h\n"
-      "fmin v13.8h, v13.8h, v0.8h\n"
-      "fmin v14.8h, v14.8h, v0.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmin v16.8h, v16.8h, v0.8h\n"
-      "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
-      "fmin v18.8h, v18.8h, v0.8h\n"
-      "fmin v19.8h, v19.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmin v12.8h, v12.8h, v1.8h\n"
+      "fmin v13.8h, v13.8h, v1.8h\n"
+      "fmin v14.8h, v14.8h, v1.8h\n"
+      "fmin v15.8h, v15.8h, v1.8h\n"
+      "fmin v16.8h, v16.8h, v1.8h\n"
+      "fmin v17.8h, v17.8h, v1.8h\n"
+      "fmin v18.8h, v18.8h, v1.8h\n"
+      "fmin v19.8h, v19.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
       "129:"  // Height 3: No activation
       "cmp x11, #0x20\n"
       "bge 146f\n"
       "tbz x11, #4, 137f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
       "tbz x11, #3, 133f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
       "tbz x11, #2, 131f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
       "tbz x11, #1, 130f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v11.h }[6], [x28]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
+      "st1 { v11.h }[6], [x9]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
       "b 145f\n"
       "130:"  // Height 3: Partial direct writeback: partial_1_28
       "tbz x11, #0, 145f\n"
-      "st1 { v11.h }[4], [x28]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
+      "st1 { v11.h }[4], [x9]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
       "b 145f\n"
       "131:"  // Height 3: Partial direct writeback: partial_2_24
       "tbz x11, #1, 132f\n"
-      "str s11, [x28], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
+      "str s11, [x9], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v11.h }[2], [x28]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
+      "st1 { v11.h }[2], [x9]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
       "b 145f\n"
       "132:"  // Height 3: Partial direct writeback: partial_1_24
       "tbz x11, #0, 145f\n"
-      "str h11, [x28, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
       "b 145f\n"
       "133:"  // Height 3: Partial direct writeback: partial_4_16
       "tbz x11, #2, 135f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
       "tbz x11, #1, 134f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v10.h }[6], [x28]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
+      "st1 { v10.h }[6], [x9]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
       "b 145f\n"
       "134:"  // Height 3: Partial direct writeback: partial_1_20
       "tbz x11, #0, 145f\n"
-      "st1 { v10.h }[4], [x28]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
+      "st1 { v10.h }[4], [x9]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
       "b 145f\n"
       "135:"  // Height 3: Partial direct writeback: partial_2_16
       "tbz x11, #1, 136f\n"
-      "str s10, [x28], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
+      "str s10, [x9], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v10.h }[2], [x28]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
+      "st1 { v10.h }[2], [x9]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
       "b 145f\n"
       "136:"  // Height 3: Partial direct writeback: partial_1_16
       "tbz x11, #0, 145f\n"
-      "str h10, [x28, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
       "b 145f\n"
       "137:"  // Height 3: Partial direct writeback: partial_8_0
       "tbz x11, #3, 141f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
       "tbz x11, #2, 139f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
       "tbz x11, #1, 138f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v9.h }[6], [x28]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
+      "st1 { v9.h }[6], [x9]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
       "b 145f\n"
       "138:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x11, #0, 145f\n"
-      "st1 { v9.h }[4], [x28]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
+      "st1 { v9.h }[4], [x9]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
       "b 145f\n"
       "139:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x11, #1, 140f\n"
-      "str s9, [x28], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
+      "str s9, [x9], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v9.h }[2], [x28]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
+      "st1 { v9.h }[2], [x9]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
       "b 145f\n"
       "140:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x11, #0, 145f\n"
-      "str h9, [x28, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
       "b 145f\n"
       "141:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x11, #2, 143f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
       "tbz x11, #1, 142f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v8.h }[6], [x28]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
+      "st1 { v8.h }[6], [x9]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
       "b 145f\n"
       "142:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x11, #0, 145f\n"
-      "st1 { v8.h }[4], [x28]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
+      "st1 { v8.h }[4], [x9]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
       "b 145f\n"
       "143:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x11, #1, 144f\n"
-      "str s8, [x28], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
+      "str s8, [x9], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
       "tbz x11, #0, 145f\n"
-      "st1 { v8.h }[2], [x28]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
+      "st1 { v8.h }[2], [x9]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
       "b 145f\n"
       "144:"  // Height 3: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
       "145:"  // Height 3: Partial direct writeback: Done
       "b 147f\n"
       "146:"  // Height 3: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "147:"  // Height 3: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 100b\n"
       "b 296f\n"
       "148:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "149:"  // Height 4: Column loop
-      "cbz x9, 150f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 150f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
       "b 169f\n"
       "150:"  // Height 4: no bias
       "tbz %x[flags], #0, 168f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "cmp x11, #0x20\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "bge 167f\n"
       "tbz x11, #4, 158f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
       "tbz x11, #3, 154f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
       "tbz x11, #2, 152f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
       "tbz x11, #1, 151f\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v11.h }[6], [x28]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
+      "ld1 { v11.h }[6], [x9]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
       "b 166f\n"
       "151:"  // Height 4: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v11.h }[4], [x28]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
+      "ld1 { v11.h }[4], [x9]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
       "b 166f\n"
       "152:"  // Height 4: Partial accumulate: partial_2_24
       "tbz x11, #1, 153f\n"
-      "ldr s11, [x28], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
+      "ldr s11, [x9], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v11.h }[2], [x28]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
+      "ld1 { v11.h }[2], [x9]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
       "b 166f\n"
       "153:"  // Height 4: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 166f\n"
-      "ldr h11, [x28, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
       "b 166f\n"
       "154:"  // Height 4: Partial accumulate: partial_4_16
       "tbz x11, #2, 156f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
       "tbz x11, #1, 155f\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v10.h }[6], [x28]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
+      "ld1 { v10.h }[6], [x9]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
       "b 166f\n"
       "155:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v10.h }[4], [x28]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
+      "ld1 { v10.h }[4], [x9]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
       "b 166f\n"
       "156:"  // Height 4: Partial accumulate: partial_2_16
       "tbz x11, #1, 157f\n"
-      "ldr s10, [x28], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
+      "ldr s10, [x9], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v10.h }[2], [x28]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
+      "ld1 { v10.h }[2], [x9]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
       "b 166f\n"
       "157:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 166f\n"
-      "ldr h10, [x28, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
       "b 166f\n"
       "158:"  // Height 4: Partial accumulate: partial_8_0
       "tbz x11, #3, 162f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
       "tbz x11, #2, 160f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
       "tbz x11, #1, 159f\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v9.h }[6], [x28]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
+      "ld1 { v9.h }[6], [x9]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
       "b 166f\n"
       "159:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v9.h }[4], [x28]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
+      "ld1 { v9.h }[4], [x9]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
       "b 166f\n"
       "160:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x11, #1, 161f\n"
-      "ldr s9, [x28], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
+      "ldr s9, [x9], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v9.h }[2], [x28]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
+      "ld1 { v9.h }[2], [x9]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
       "b 166f\n"
       "161:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 166f\n"
-      "ldr h9, [x28, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
       "b 166f\n"
       "162:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x11, #2, 164f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
       "tbz x11, #1, 163f\n"
-      "mov x19, #0xc\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v8.h }[6], [x28]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
+      "ld1 { v8.h }[6], [x9]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
       "b 166f\n"
       "163:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v8.h }[4], [x28]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
+      "ld1 { v8.h }[4], [x9]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
       "b 166f\n"
       "164:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x11, #1, 165f\n"
-      "ldr s8, [x28], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
       "tbz x11, #0, 166f\n"
-      "ld1 { v8.h }[2], [x28]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
+      "ld1 { v8.h }[2], [x9]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
       "b 166f\n"
       "165:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h12, [x24, #0x0]\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
+      "ldr h8, [x9, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
       "166:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 169f\n"
       "167:"  // Height 4: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "b 169f\n"
       "168:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2216,69 +2216,69 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "169:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "170:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 171f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 172f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 172f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 172f\n"
       "171:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "172:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 175f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 174f\n"
       "173:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "cmp x27, #0x10\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -2419,40 +2419,40 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v22.8h, v6.8h, v3.h[7]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 173b\n"
       "174:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
@@ -2598,16 +2598,16 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v19.8h, v7.8h, v2.h[7]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "175:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 177f\n"
+      "cbz x27, 177f\n"
       "176:"  // Height 4: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
@@ -2626,570 +2626,570 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v15.8h, v7.8h, v1.h[0]\n"
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
-      "cbnz x26, 176b\n"
+      "cbnz x27, 176b\n"
       "177:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 170b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 178f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmin v12.8h, v12.8h, v0.8h\n"
-      "fmin v13.8h, v13.8h, v0.8h\n"
-      "fmin v14.8h, v14.8h, v0.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmin v16.8h, v16.8h, v0.8h\n"
-      "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
-      "fmin v18.8h, v18.8h, v0.8h\n"
-      "fmin v19.8h, v19.8h, v0.8h\n"
-      "fmin v20.8h, v20.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmin v21.8h, v21.8h, v0.8h\n"
-      "fmin v22.8h, v22.8h, v0.8h\n"
-      "fmin v23.8h, v23.8h, v0.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmin v12.8h, v12.8h, v1.8h\n"
+      "fmin v13.8h, v13.8h, v1.8h\n"
+      "fmin v14.8h, v14.8h, v1.8h\n"
+      "fmin v15.8h, v15.8h, v1.8h\n"
+      "fmin v16.8h, v16.8h, v1.8h\n"
+      "fmin v17.8h, v17.8h, v1.8h\n"
+      "fmin v18.8h, v18.8h, v1.8h\n"
+      "fmin v19.8h, v19.8h, v1.8h\n"
+      "fmin v20.8h, v20.8h, v1.8h\n"
+      "fmin v21.8h, v21.8h, v1.8h\n"
+      "fmin v22.8h, v22.8h, v1.8h\n"
+      "fmin v23.8h, v23.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
       "178:"  // Height 4: No activation
       "cmp x11, #0x20\n"
       "bge 195f\n"
       "tbz x11, #4, 186f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
       "tbz x11, #3, 182f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
       "tbz x11, #2, 180f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
       "tbz x11, #1, 179f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v11.h }[6], [x28]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
+      "st1 { v11.h }[6], [x9]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
       "b 194f\n"
       "179:"  // Height 4: Partial direct writeback: partial_1_28
       "tbz x11, #0, 194f\n"
-      "st1 { v11.h }[4], [x28]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
+      "st1 { v11.h }[4], [x9]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
       "b 194f\n"
       "180:"  // Height 4: Partial direct writeback: partial_2_24
       "tbz x11, #1, 181f\n"
-      "str s11, [x28], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
+      "str s11, [x9], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v11.h }[2], [x28]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
+      "st1 { v11.h }[2], [x9]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
       "b 194f\n"
       "181:"  // Height 4: Partial direct writeback: partial_1_24
       "tbz x11, #0, 194f\n"
-      "str h11, [x28, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
       "b 194f\n"
       "182:"  // Height 4: Partial direct writeback: partial_4_16
       "tbz x11, #2, 184f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
       "tbz x11, #1, 183f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v10.h }[6], [x28]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
+      "st1 { v10.h }[6], [x9]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
       "b 194f\n"
       "183:"  // Height 4: Partial direct writeback: partial_1_20
       "tbz x11, #0, 194f\n"
-      "st1 { v10.h }[4], [x28]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
+      "st1 { v10.h }[4], [x9]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
       "b 194f\n"
       "184:"  // Height 4: Partial direct writeback: partial_2_16
       "tbz x11, #1, 185f\n"
-      "str s10, [x28], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
+      "str s10, [x9], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v10.h }[2], [x28]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
+      "st1 { v10.h }[2], [x9]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
       "b 194f\n"
       "185:"  // Height 4: Partial direct writeback: partial_1_16
       "tbz x11, #0, 194f\n"
-      "str h10, [x28, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
       "b 194f\n"
       "186:"  // Height 4: Partial direct writeback: partial_8_0
       "tbz x11, #3, 190f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
       "tbz x11, #2, 188f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
       "tbz x11, #1, 187f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v9.h }[6], [x28]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
+      "st1 { v9.h }[6], [x9]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
       "b 194f\n"
       "187:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x11, #0, 194f\n"
-      "st1 { v9.h }[4], [x28]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
+      "st1 { v9.h }[4], [x9]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
       "b 194f\n"
       "188:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x11, #1, 189f\n"
-      "str s9, [x28], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
+      "str s9, [x9], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v9.h }[2], [x28]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
+      "st1 { v9.h }[2], [x9]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
       "b 194f\n"
       "189:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x11, #0, 194f\n"
-      "str h9, [x28, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
       "b 194f\n"
       "190:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x11, #2, 192f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
       "tbz x11, #1, 191f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v8.h }[6], [x28]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
+      "st1 { v8.h }[6], [x9]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
       "b 194f\n"
       "191:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x11, #0, 194f\n"
-      "st1 { v8.h }[4], [x28]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
+      "st1 { v8.h }[4], [x9]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
       "b 194f\n"
       "192:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x11, #1, 193f\n"
-      "str s8, [x28], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
+      "str s8, [x9], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
       "tbz x11, #0, 194f\n"
-      "st1 { v8.h }[2], [x28]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
+      "st1 { v8.h }[2], [x9]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
       "b 194f\n"
       "193:"  // Height 4: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
       "194:"  // Height 4: Partial direct writeback: Done
       "b 196f\n"
       "195:"  // Height 4: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
       "196:"  // Height 4: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 149b\n"
       "b 296f\n"
       "197:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "198:"  // Height 5: Column loop
-      "cbz x9, 199f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 199f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
       "b 218f\n"
       "199:"  // Height 5: no bias
       "tbz %x[flags], #0, 217f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "cmp x11, #0x20\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "bge 216f\n"
       "tbz x11, #4, 207f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
-      "ld1 { v25.8h }, [x21], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
+      "ld1 { v25.8h }, [x22], #0x10\n"
       "tbz x11, #3, 203f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
-      "ld1 { v26.8h }, [x21], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
+      "ld1 { v26.8h }, [x22], #0x10\n"
       "tbz x11, #2, 201f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
       "tbz x11, #1, 200f\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
-      "ld1 { v27.s }[2], [x21], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
+      "ld1 { v27.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v11.h }[6], [x28]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
-      "ld1 { v27.h }[6], [x21]\n"
+      "ld1 { v11.h }[6], [x9]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
+      "ld1 { v27.h }[6], [x22]\n"
       "b 215f\n"
       "200:"  // Height 5: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v11.h }[4], [x28]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
-      "ld1 { v27.h }[4], [x21]\n"
+      "ld1 { v11.h }[4], [x9]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
+      "ld1 { v27.h }[4], [x22]\n"
       "b 215f\n"
       "201:"  // Height 5: Partial accumulate: partial_2_24
       "tbz x11, #1, 202f\n"
-      "ldr s11, [x28], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
-      "ldr s27, [x21], #0x4\n"
+      "ldr s11, [x9], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
+      "ldr s27, [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v11.h }[2], [x28]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
-      "ld1 { v27.h }[2], [x21]\n"
+      "ld1 { v11.h }[2], [x9]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
+      "ld1 { v27.h }[2], [x22]\n"
       "b 215f\n"
       "202:"  // Height 5: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 215f\n"
-      "ldr h11, [x28, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
-      "ldr h27, [x21, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
+      "ldr h27, [x22, #0x0]\n"
       "b 215f\n"
       "203:"  // Height 5: Partial accumulate: partial_4_16
       "tbz x11, #2, 205f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
       "tbz x11, #1, 204f\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
-      "ld1 { v26.s }[2], [x21], #0x4\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
+      "ld1 { v26.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v10.h }[6], [x28]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
-      "ld1 { v26.h }[6], [x21]\n"
+      "ld1 { v10.h }[6], [x9]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
+      "ld1 { v26.h }[6], [x22]\n"
       "b 215f\n"
       "204:"  // Height 5: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v10.h }[4], [x28]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
-      "ld1 { v26.h }[4], [x21]\n"
+      "ld1 { v10.h }[4], [x9]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
+      "ld1 { v26.h }[4], [x22]\n"
       "b 215f\n"
       "205:"  // Height 5: Partial accumulate: partial_2_16
       "tbz x11, #1, 206f\n"
-      "ldr s10, [x28], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
-      "ldr s26, [x21], #0x4\n"
+      "ldr s10, [x9], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
+      "ldr s26, [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v10.h }[2], [x28]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
-      "ld1 { v26.h }[2], [x21]\n"
+      "ld1 { v10.h }[2], [x9]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
+      "ld1 { v26.h }[2], [x22]\n"
       "b 215f\n"
       "206:"  // Height 5: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 215f\n"
-      "ldr h10, [x28, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
-      "ldr h26, [x21, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
+      "ldr h26, [x22, #0x0]\n"
       "b 215f\n"
       "207:"  // Height 5: Partial accumulate: partial_8_0
       "tbz x11, #3, 211f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
       "tbz x11, #2, 209f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x11, #1, 208f\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
-      "ld1 { v25.s }[2], [x21], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
+      "ld1 { v25.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v9.h }[6], [x28]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
-      "ld1 { v25.h }[6], [x21]\n"
+      "ld1 { v9.h }[6], [x9]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
+      "ld1 { v25.h }[6], [x22]\n"
       "b 215f\n"
       "208:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v9.h }[4], [x28]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
-      "ld1 { v25.h }[4], [x21]\n"
+      "ld1 { v9.h }[4], [x9]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
+      "ld1 { v25.h }[4], [x22]\n"
       "b 215f\n"
       "209:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x11, #1, 210f\n"
-      "ldr s9, [x28], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
-      "ldr s25, [x21], #0x4\n"
+      "ldr s9, [x9], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
+      "ldr s25, [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v9.h }[2], [x28]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
-      "ld1 { v25.h }[2], [x21]\n"
+      "ld1 { v9.h }[2], [x9]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
+      "ld1 { v25.h }[2], [x22]\n"
       "b 215f\n"
       "210:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 215f\n"
-      "ldr h9, [x28, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
-      "ldr h25, [x21, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
+      "ldr h25, [x22, #0x0]\n"
       "b 215f\n"
       "211:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x11, #2, 213f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
       "tbz x11, #1, 212f\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
-      "ld1 { v24.s }[2], [x21], #0x4\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
+      "ld1 { v24.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v8.h }[6], [x28]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
-      "ld1 { v24.h }[6], [x21]\n"
+      "ld1 { v8.h }[6], [x9]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
+      "ld1 { v24.h }[6], [x22]\n"
       "b 215f\n"
       "212:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v8.h }[4], [x28]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
-      "ld1 { v24.h }[4], [x21]\n"
+      "ld1 { v8.h }[4], [x9]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
+      "ld1 { v24.h }[4], [x22]\n"
       "b 215f\n"
       "213:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x11, #1, 214f\n"
-      "ldr s8, [x28], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
-      "ldr s24, [x21], #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
+      "ldr s24, [x22], #0x4\n"
       "tbz x11, #0, 215f\n"
-      "ld1 { v8.h }[2], [x28]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
-      "ld1 { v24.h }[2], [x21]\n"
+      "ld1 { v8.h }[2], [x9]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
+      "ld1 { v24.h }[2], [x22]\n"
       "b 215f\n"
       "214:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h12, [x24, #0x0]\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
-      "ldr h24, [x21, #0x0]\n"
+      "ldr h8, [x9, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
+      "ldr h24, [x22, #0x0]\n"
       "215:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 218f\n"
       "216:"  // Height 5: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
       "b 218f\n"
       "217:"  // Height 5: no accumulate
       "movi v8.16b, #0x0\n"
@@ -3213,74 +3213,74 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "218:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "219:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 220f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 221f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 221f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 221f\n"
       "220:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "221:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 224f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 223f\n"
       "222:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x8\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x10\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
@@ -3453,45 +3453,45 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v26.8h, v6.8h, v4.h[7]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 222b\n"
       "223:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
+      "add x22, x22, #0x10\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x8\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.8h, v7.8h, v0.h[0]\n"
@@ -3668,18 +3668,18 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v23.8h, v7.8h, v3.h[7]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "224:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 226f\n"
+      "cbz x27, 226f\n"
       "225:"  // Height 5: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
@@ -3701,660 +3701,660 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v19.8h, v7.8h, v2.h[0]\n"
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
-      "cbnz x26, 225b\n"
+      "cbnz x27, 225b\n"
       "226:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 219b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #1\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #1\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 227f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmin v12.8h, v12.8h, v0.8h\n"
-      "fmin v13.8h, v13.8h, v0.8h\n"
-      "fmin v14.8h, v14.8h, v0.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmin v16.8h, v16.8h, v0.8h\n"
-      "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
-      "fmin v18.8h, v18.8h, v0.8h\n"
-      "fmin v19.8h, v19.8h, v0.8h\n"
-      "fmin v20.8h, v20.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmin v21.8h, v21.8h, v0.8h\n"
-      "fmin v22.8h, v22.8h, v0.8h\n"
-      "fmin v23.8h, v23.8h, v0.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
-      "fmin v24.8h, v24.8h, v0.8h\n"
-      "fmin v25.8h, v25.8h, v0.8h\n"
-      "fmin v26.8h, v26.8h, v0.8h\n"
-      "fmax v24.8h, v24.8h, v1.8h\n"
-      "fmax v25.8h, v25.8h, v1.8h\n"
-      "fmax v26.8h, v26.8h, v1.8h\n"
-      "fmin v27.8h, v27.8h, v0.8h\n"
-      "fmax v27.8h, v27.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmin v12.8h, v12.8h, v1.8h\n"
+      "fmin v13.8h, v13.8h, v1.8h\n"
+      "fmin v14.8h, v14.8h, v1.8h\n"
+      "fmin v15.8h, v15.8h, v1.8h\n"
+      "fmin v16.8h, v16.8h, v1.8h\n"
+      "fmin v17.8h, v17.8h, v1.8h\n"
+      "fmin v18.8h, v18.8h, v1.8h\n"
+      "fmin v19.8h, v19.8h, v1.8h\n"
+      "fmin v20.8h, v20.8h, v1.8h\n"
+      "fmin v21.8h, v21.8h, v1.8h\n"
+      "fmin v22.8h, v22.8h, v1.8h\n"
+      "fmin v23.8h, v23.8h, v1.8h\n"
+      "fmin v24.8h, v24.8h, v1.8h\n"
+      "fmin v25.8h, v25.8h, v1.8h\n"
+      "fmin v26.8h, v26.8h, v1.8h\n"
+      "fmin v27.8h, v27.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
+      "fmax v24.8h, v24.8h, v0.8h\n"
+      "fmax v25.8h, v25.8h, v0.8h\n"
+      "fmax v26.8h, v26.8h, v0.8h\n"
+      "fmax v27.8h, v27.8h, v0.8h\n"
       "227:"  // Height 5: No activation
       "cmp x11, #0x20\n"
       "bge 244f\n"
       "tbz x11, #4, 235f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v25.8h }, [x21], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v25.8h }, [x22], #0x10\n"
       "tbz x11, #3, 231f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
-      "st1 { v26.8h }, [x21], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
+      "st1 { v26.8h }, [x22], #0x10\n"
       "tbz x11, #2, 229f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
       "tbz x11, #1, 228f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
-      "st1 { v27.s }[2], [x21], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
+      "st1 { v27.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v11.h }[6], [x28]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
-      "st1 { v27.h }[6], [x21]\n"
+      "st1 { v11.h }[6], [x9]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
+      "st1 { v27.h }[6], [x22]\n"
       "b 243f\n"
       "228:"  // Height 5: Partial direct writeback: partial_1_28
       "tbz x11, #0, 243f\n"
-      "st1 { v11.h }[4], [x28]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
-      "st1 { v27.h }[4], [x21]\n"
+      "st1 { v11.h }[4], [x9]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
+      "st1 { v27.h }[4], [x22]\n"
       "b 243f\n"
       "229:"  // Height 5: Partial direct writeback: partial_2_24
       "tbz x11, #1, 230f\n"
-      "str s11, [x28], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
-      "str s27, [x21], #0x4\n"
+      "str s11, [x9], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
+      "str s27, [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v11.h }[2], [x28]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
-      "st1 { v27.h }[2], [x21]\n"
+      "st1 { v11.h }[2], [x9]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
+      "st1 { v27.h }[2], [x22]\n"
       "b 243f\n"
       "230:"  // Height 5: Partial direct writeback: partial_1_24
       "tbz x11, #0, 243f\n"
-      "str h11, [x28, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
-      "str h27, [x21, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
+      "str h27, [x22, #0x0]\n"
       "b 243f\n"
       "231:"  // Height 5: Partial direct writeback: partial_4_16
       "tbz x11, #2, 233f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
       "tbz x11, #1, 232f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
-      "st1 { v26.s }[2], [x21], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
+      "st1 { v26.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v10.h }[6], [x28]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
-      "st1 { v26.h }[6], [x21]\n"
+      "st1 { v10.h }[6], [x9]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
+      "st1 { v26.h }[6], [x22]\n"
       "b 243f\n"
       "232:"  // Height 5: Partial direct writeback: partial_1_20
       "tbz x11, #0, 243f\n"
-      "st1 { v10.h }[4], [x28]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
-      "st1 { v26.h }[4], [x21]\n"
+      "st1 { v10.h }[4], [x9]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
+      "st1 { v26.h }[4], [x22]\n"
       "b 243f\n"
       "233:"  // Height 5: Partial direct writeback: partial_2_16
       "tbz x11, #1, 234f\n"
-      "str s10, [x28], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
-      "str s26, [x21], #0x4\n"
+      "str s10, [x9], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
+      "str s26, [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v10.h }[2], [x28]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
-      "st1 { v26.h }[2], [x21]\n"
+      "st1 { v10.h }[2], [x9]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
+      "st1 { v26.h }[2], [x22]\n"
       "b 243f\n"
       "234:"  // Height 5: Partial direct writeback: partial_1_16
       "tbz x11, #0, 243f\n"
-      "str h10, [x28, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
-      "str h26, [x21, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
+      "str h26, [x22, #0x0]\n"
       "b 243f\n"
       "235:"  // Height 5: Partial direct writeback: partial_8_0
       "tbz x11, #3, 239f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
       "tbz x11, #2, 237f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x11, #1, 236f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
-      "st1 { v25.s }[2], [x21], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
+      "st1 { v25.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v9.h }[6], [x28]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
-      "st1 { v25.h }[6], [x21]\n"
+      "st1 { v9.h }[6], [x9]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
+      "st1 { v25.h }[6], [x22]\n"
       "b 243f\n"
       "236:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x11, #0, 243f\n"
-      "st1 { v9.h }[4], [x28]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
-      "st1 { v25.h }[4], [x21]\n"
+      "st1 { v9.h }[4], [x9]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
+      "st1 { v25.h }[4], [x22]\n"
       "b 243f\n"
       "237:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x11, #1, 238f\n"
-      "str s9, [x28], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
-      "str s25, [x21], #0x4\n"
+      "str s9, [x9], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
+      "str s25, [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v9.h }[2], [x28]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
-      "st1 { v25.h }[2], [x21]\n"
+      "st1 { v9.h }[2], [x9]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
+      "st1 { v25.h }[2], [x22]\n"
       "b 243f\n"
       "238:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x11, #0, 243f\n"
-      "str h9, [x28, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
-      "str h25, [x21, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
+      "str h25, [x22, #0x0]\n"
       "b 243f\n"
       "239:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x11, #2, 241f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x11, #1, 240f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
-      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
+      "st1 { v24.s }[2], [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v8.h }[6], [x28]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
-      "st1 { v24.h }[6], [x21]\n"
+      "st1 { v8.h }[6], [x9]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
+      "st1 { v24.h }[6], [x22]\n"
       "b 243f\n"
       "240:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x11, #0, 243f\n"
-      "st1 { v8.h }[4], [x28]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
-      "st1 { v24.h }[4], [x21]\n"
+      "st1 { v8.h }[4], [x9]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
+      "st1 { v24.h }[4], [x22]\n"
       "b 243f\n"
       "241:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x11, #1, 242f\n"
-      "str s8, [x28], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
-      "str s24, [x21], #0x4\n"
+      "str s8, [x9], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
+      "str s24, [x22], #0x4\n"
       "tbz x11, #0, 243f\n"
-      "st1 { v8.h }[2], [x28]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
-      "st1 { v24.h }[2], [x21]\n"
+      "st1 { v8.h }[2], [x9]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
+      "st1 { v24.h }[2], [x22]\n"
       "b 243f\n"
       "242:"  // Height 5: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
-      "str h24, [x21, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
+      "str h24, [x22, #0x0]\n"
       "243:"  // Height 5: Partial direct writeback: Done
       "b 245f\n"
       "244:"  // Height 5: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "245:"  // Height 5: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 198b\n"
       "b 296f\n"
       "246:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0xc\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0xc\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "247:"  // Height 6: Column loop
-      "cbz x9, 248f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 248f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
-      "mov v28.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
+      "mov v28.16b, v8.16b\n"
       "mov v29.16b, v9.16b\n"
       "mov v30.16b, v10.16b\n"
       "mov v31.16b, v11.16b\n"
       "b 267f\n"
       "248:"  // Height 6: no bias
       "tbz %x[flags], #0, 266f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "cmp x11, #0x20\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "bge 265f\n"
       "tbz x11, #4, 256f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v9.8h }, [x28], #0x10\n"
-      "ld1 { v13.8h }, [x24], #0x10\n"
-      "ld1 { v17.8h }, [x23], #0x10\n"
-      "ld1 { v21.8h }, [x22], #0x10\n"
-      "ld1 { v25.8h }, [x21], #0x10\n"
-      "ld1 { v28.8h }, [x20], #0x10\n"
-      "ld1 { v29.8h }, [x20], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v28.8h }, [x21], #0x10\n"
+      "ld1 { v9.8h }, [x9], #0x10\n"
+      "ld1 { v13.8h }, [x25], #0x10\n"
+      "ld1 { v17.8h }, [x24], #0x10\n"
+      "ld1 { v21.8h }, [x23], #0x10\n"
+      "ld1 { v25.8h }, [x22], #0x10\n"
+      "ld1 { v29.8h }, [x21], #0x10\n"
       "tbz x11, #3, 252f\n"
-      "ld1 { v10.8h }, [x28], #0x10\n"
-      "ld1 { v14.8h }, [x24], #0x10\n"
-      "ld1 { v18.8h }, [x23], #0x10\n"
-      "ld1 { v22.8h }, [x22], #0x10\n"
-      "ld1 { v26.8h }, [x21], #0x10\n"
-      "ld1 { v30.8h }, [x20], #0x10\n"
+      "ld1 { v10.8h }, [x9], #0x10\n"
+      "ld1 { v14.8h }, [x25], #0x10\n"
+      "ld1 { v18.8h }, [x24], #0x10\n"
+      "ld1 { v22.8h }, [x23], #0x10\n"
+      "ld1 { v26.8h }, [x22], #0x10\n"
+      "ld1 { v30.8h }, [x21], #0x10\n"
       "tbz x11, #2, 250f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x11, #1, 249f\n"
-      "ld1 { v11.s }[2], [x28], #0x4\n"
-      "mov x19, #0x3c\n"
-      "ld1 { v15.s }[2], [x24], #0x4\n"
-      "ld1 { v19.s }[2], [x23], #0x4\n"
-      "ld1 { v23.s }[2], [x22], #0x4\n"
-      "ld1 { v27.s }[2], [x21], #0x4\n"
-      "ld1 { v31.s }[2], [x20], #0x4\n"
+      "ld1 { v11.s }[2], [x9], #0x4\n"
+      "ld1 { v15.s }[2], [x25], #0x4\n"
+      "mov x20, #0x3c\n"
+      "ld1 { v19.s }[2], [x24], #0x4\n"
+      "ld1 { v23.s }[2], [x23], #0x4\n"
+      "ld1 { v27.s }[2], [x22], #0x4\n"
+      "ld1 { v31.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v11.h }[6], [x28]\n"
-      "ld1 { v15.h }[6], [x24]\n"
-      "ld1 { v19.h }[6], [x23]\n"
-      "ld1 { v23.h }[6], [x22]\n"
-      "ld1 { v27.h }[6], [x21]\n"
-      "ld1 { v31.h }[6], [x20]\n"
+      "ld1 { v11.h }[6], [x9]\n"
+      "ld1 { v15.h }[6], [x25]\n"
+      "ld1 { v19.h }[6], [x24]\n"
+      "ld1 { v23.h }[6], [x23]\n"
+      "ld1 { v27.h }[6], [x22]\n"
+      "ld1 { v31.h }[6], [x21]\n"
       "b 264f\n"
       "249:"  // Height 6: Partial accumulate: partial_1_28
-      "mov x19, #0x38\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v11.h }[4], [x28]\n"
-      "ld1 { v15.h }[4], [x24]\n"
-      "ld1 { v19.h }[4], [x23]\n"
-      "ld1 { v23.h }[4], [x22]\n"
-      "ld1 { v27.h }[4], [x21]\n"
-      "ld1 { v31.h }[4], [x20]\n"
+      "ld1 { v11.h }[4], [x9]\n"
+      "ld1 { v15.h }[4], [x25]\n"
+      "ld1 { v19.h }[4], [x24]\n"
+      "ld1 { v23.h }[4], [x23]\n"
+      "ld1 { v27.h }[4], [x22]\n"
+      "ld1 { v31.h }[4], [x21]\n"
       "b 264f\n"
       "250:"  // Height 6: Partial accumulate: partial_2_24
       "tbz x11, #1, 251f\n"
-      "ldr s11, [x28], #0x4\n"
-      "ldr s15, [x24], #0x4\n"
-      "mov x19, #0x34\n"
-      "ldr s19, [x23], #0x4\n"
-      "ldr s23, [x22], #0x4\n"
-      "ldr s27, [x21], #0x4\n"
-      "ldr s31, [x20], #0x4\n"
+      "ldr s11, [x9], #0x4\n"
+      "ldr s15, [x25], #0x4\n"
+      "mov x20, #0x34\n"
+      "ldr s19, [x24], #0x4\n"
+      "ldr s23, [x23], #0x4\n"
+      "ldr s27, [x22], #0x4\n"
+      "ldr s31, [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v11.h }[2], [x28]\n"
-      "ld1 { v15.h }[2], [x24]\n"
-      "ld1 { v19.h }[2], [x23]\n"
-      "ld1 { v23.h }[2], [x22]\n"
-      "ld1 { v27.h }[2], [x21]\n"
-      "ld1 { v31.h }[2], [x20]\n"
+      "ld1 { v11.h }[2], [x9]\n"
+      "ld1 { v15.h }[2], [x25]\n"
+      "ld1 { v19.h }[2], [x24]\n"
+      "ld1 { v23.h }[2], [x23]\n"
+      "ld1 { v27.h }[2], [x22]\n"
+      "ld1 { v31.h }[2], [x21]\n"
       "b 264f\n"
       "251:"  // Height 6: Partial accumulate: partial_1_24
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 264f\n"
-      "ldr h11, [x28, #0x0]\n"
-      "ldr h15, [x24, #0x0]\n"
-      "ldr h19, [x23, #0x0]\n"
-      "ldr h23, [x22, #0x0]\n"
-      "ldr h27, [x21, #0x0]\n"
-      "ldr h31, [x20, #0x0]\n"
+      "ldr h11, [x9, #0x0]\n"
+      "ldr h15, [x25, #0x0]\n"
+      "ldr h19, [x24, #0x0]\n"
+      "ldr h23, [x23, #0x0]\n"
+      "ldr h27, [x22, #0x0]\n"
+      "ldr h31, [x21, #0x0]\n"
       "b 264f\n"
       "252:"  // Height 6: Partial accumulate: partial_4_16
       "tbz x11, #2, 254f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x11, #1, 253f\n"
-      "ld1 { v10.s }[2], [x28], #0x4\n"
-      "mov x19, #0x2c\n"
-      "ld1 { v14.s }[2], [x24], #0x4\n"
-      "ld1 { v18.s }[2], [x23], #0x4\n"
-      "ld1 { v22.s }[2], [x22], #0x4\n"
-      "ld1 { v26.s }[2], [x21], #0x4\n"
-      "ld1 { v30.s }[2], [x20], #0x4\n"
+      "ld1 { v10.s }[2], [x9], #0x4\n"
+      "ld1 { v14.s }[2], [x25], #0x4\n"
+      "mov x20, #0x2c\n"
+      "ld1 { v18.s }[2], [x24], #0x4\n"
+      "ld1 { v22.s }[2], [x23], #0x4\n"
+      "ld1 { v26.s }[2], [x22], #0x4\n"
+      "ld1 { v30.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v10.h }[6], [x28]\n"
-      "ld1 { v14.h }[6], [x24]\n"
-      "ld1 { v18.h }[6], [x23]\n"
-      "ld1 { v22.h }[6], [x22]\n"
-      "ld1 { v26.h }[6], [x21]\n"
-      "ld1 { v30.h }[6], [x20]\n"
+      "ld1 { v10.h }[6], [x9]\n"
+      "ld1 { v14.h }[6], [x25]\n"
+      "ld1 { v18.h }[6], [x24]\n"
+      "ld1 { v22.h }[6], [x23]\n"
+      "ld1 { v26.h }[6], [x22]\n"
+      "ld1 { v30.h }[6], [x21]\n"
       "b 264f\n"
       "253:"  // Height 6: Partial accumulate: partial_1_20
-      "mov x19, #0x28\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v10.h }[4], [x28]\n"
-      "ld1 { v14.h }[4], [x24]\n"
-      "ld1 { v18.h }[4], [x23]\n"
-      "ld1 { v22.h }[4], [x22]\n"
-      "ld1 { v26.h }[4], [x21]\n"
-      "ld1 { v30.h }[4], [x20]\n"
+      "ld1 { v10.h }[4], [x9]\n"
+      "ld1 { v14.h }[4], [x25]\n"
+      "ld1 { v18.h }[4], [x24]\n"
+      "ld1 { v22.h }[4], [x23]\n"
+      "ld1 { v26.h }[4], [x22]\n"
+      "ld1 { v30.h }[4], [x21]\n"
       "b 264f\n"
       "254:"  // Height 6: Partial accumulate: partial_2_16
       "tbz x11, #1, 255f\n"
-      "ldr s10, [x28], #0x4\n"
-      "ldr s14, [x24], #0x4\n"
-      "mov x19, #0x24\n"
-      "ldr s18, [x23], #0x4\n"
-      "ldr s22, [x22], #0x4\n"
-      "ldr s26, [x21], #0x4\n"
-      "ldr s30, [x20], #0x4\n"
+      "ldr s10, [x9], #0x4\n"
+      "ldr s14, [x25], #0x4\n"
+      "mov x20, #0x24\n"
+      "ldr s18, [x24], #0x4\n"
+      "ldr s22, [x23], #0x4\n"
+      "ldr s26, [x22], #0x4\n"
+      "ldr s30, [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v10.h }[2], [x28]\n"
-      "ld1 { v14.h }[2], [x24]\n"
-      "ld1 { v18.h }[2], [x23]\n"
-      "ld1 { v22.h }[2], [x22]\n"
-      "ld1 { v26.h }[2], [x21]\n"
-      "ld1 { v30.h }[2], [x20]\n"
+      "ld1 { v10.h }[2], [x9]\n"
+      "ld1 { v14.h }[2], [x25]\n"
+      "ld1 { v18.h }[2], [x24]\n"
+      "ld1 { v22.h }[2], [x23]\n"
+      "ld1 { v26.h }[2], [x22]\n"
+      "ld1 { v30.h }[2], [x21]\n"
       "b 264f\n"
       "255:"  // Height 6: Partial accumulate: partial_1_16
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 264f\n"
-      "ldr h10, [x28, #0x0]\n"
-      "ldr h14, [x24, #0x0]\n"
-      "ldr h18, [x23, #0x0]\n"
-      "ldr h22, [x22, #0x0]\n"
-      "ldr h26, [x21, #0x0]\n"
-      "ldr h30, [x20, #0x0]\n"
+      "ldr h10, [x9, #0x0]\n"
+      "ldr h14, [x25, #0x0]\n"
+      "ldr h18, [x24, #0x0]\n"
+      "ldr h22, [x23, #0x0]\n"
+      "ldr h26, [x22, #0x0]\n"
+      "ldr h30, [x21, #0x0]\n"
       "b 264f\n"
       "256:"  // Height 6: Partial accumulate: partial_8_0
       "tbz x11, #3, 260f\n"
-      "ld1 { v8.8h }, [x28], #0x10\n"
-      "ld1 { v12.8h }, [x24], #0x10\n"
-      "ld1 { v16.8h }, [x23], #0x10\n"
-      "ld1 { v20.8h }, [x22], #0x10\n"
-      "ld1 { v24.8h }, [x21], #0x10\n"
-      "ld1 { v28.8h }, [x20], #0x10\n"
+      "ld1 { v8.8h }, [x9], #0x10\n"
+      "ld1 { v12.8h }, [x25], #0x10\n"
+      "ld1 { v16.8h }, [x24], #0x10\n"
+      "ld1 { v20.8h }, [x23], #0x10\n"
+      "ld1 { v24.8h }, [x22], #0x10\n"
+      "ld1 { v28.8h }, [x21], #0x10\n"
       "tbz x11, #2, 258f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x11, #1, 257f\n"
-      "ld1 { v9.s }[2], [x28], #0x4\n"
-      "mov x19, #0x1c\n"
-      "ld1 { v13.s }[2], [x24], #0x4\n"
-      "ld1 { v17.s }[2], [x23], #0x4\n"
-      "ld1 { v21.s }[2], [x22], #0x4\n"
-      "ld1 { v25.s }[2], [x21], #0x4\n"
-      "ld1 { v29.s }[2], [x20], #0x4\n"
+      "ld1 { v9.s }[2], [x9], #0x4\n"
+      "ld1 { v13.s }[2], [x25], #0x4\n"
+      "mov x20, #0x1c\n"
+      "ld1 { v17.s }[2], [x24], #0x4\n"
+      "ld1 { v21.s }[2], [x23], #0x4\n"
+      "ld1 { v25.s }[2], [x22], #0x4\n"
+      "ld1 { v29.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v9.h }[6], [x28]\n"
-      "ld1 { v13.h }[6], [x24]\n"
-      "ld1 { v17.h }[6], [x23]\n"
-      "ld1 { v21.h }[6], [x22]\n"
-      "ld1 { v25.h }[6], [x21]\n"
-      "ld1 { v29.h }[6], [x20]\n"
+      "ld1 { v9.h }[6], [x9]\n"
+      "ld1 { v13.h }[6], [x25]\n"
+      "ld1 { v17.h }[6], [x24]\n"
+      "ld1 { v21.h }[6], [x23]\n"
+      "ld1 { v25.h }[6], [x22]\n"
+      "ld1 { v29.h }[6], [x21]\n"
       "b 264f\n"
       "257:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x18\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v9.h }[4], [x28]\n"
-      "ld1 { v13.h }[4], [x24]\n"
-      "ld1 { v17.h }[4], [x23]\n"
-      "ld1 { v21.h }[4], [x22]\n"
-      "ld1 { v25.h }[4], [x21]\n"
-      "ld1 { v29.h }[4], [x20]\n"
+      "ld1 { v9.h }[4], [x9]\n"
+      "ld1 { v13.h }[4], [x25]\n"
+      "ld1 { v17.h }[4], [x24]\n"
+      "ld1 { v21.h }[4], [x23]\n"
+      "ld1 { v25.h }[4], [x22]\n"
+      "ld1 { v29.h }[4], [x21]\n"
       "b 264f\n"
       "258:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x11, #1, 259f\n"
-      "ldr s9, [x28], #0x4\n"
-      "ldr s13, [x24], #0x4\n"
-      "mov x19, #0x14\n"
-      "ldr s17, [x23], #0x4\n"
-      "ldr s21, [x22], #0x4\n"
-      "ldr s25, [x21], #0x4\n"
-      "ldr s29, [x20], #0x4\n"
+      "ldr s9, [x9], #0x4\n"
+      "ldr s13, [x25], #0x4\n"
+      "mov x20, #0x14\n"
+      "ldr s17, [x24], #0x4\n"
+      "ldr s21, [x23], #0x4\n"
+      "ldr s25, [x22], #0x4\n"
+      "ldr s29, [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v9.h }[2], [x28]\n"
-      "ld1 { v13.h }[2], [x24]\n"
-      "ld1 { v17.h }[2], [x23]\n"
-      "ld1 { v21.h }[2], [x22]\n"
-      "ld1 { v25.h }[2], [x21]\n"
-      "ld1 { v29.h }[2], [x20]\n"
+      "ld1 { v9.h }[2], [x9]\n"
+      "ld1 { v13.h }[2], [x25]\n"
+      "ld1 { v17.h }[2], [x24]\n"
+      "ld1 { v21.h }[2], [x23]\n"
+      "ld1 { v25.h }[2], [x22]\n"
+      "ld1 { v29.h }[2], [x21]\n"
       "b 264f\n"
       "259:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 264f\n"
-      "ldr h9, [x28, #0x0]\n"
-      "ldr h13, [x24, #0x0]\n"
-      "ldr h17, [x23, #0x0]\n"
-      "ldr h21, [x22, #0x0]\n"
-      "ldr h25, [x21, #0x0]\n"
-      "ldr h29, [x20, #0x0]\n"
+      "ldr h9, [x9, #0x0]\n"
+      "ldr h13, [x25, #0x0]\n"
+      "ldr h17, [x24, #0x0]\n"
+      "ldr h21, [x23, #0x0]\n"
+      "ldr h25, [x22, #0x0]\n"
+      "ldr h29, [x21, #0x0]\n"
       "b 264f\n"
       "260:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x11, #2, 262f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x11, #1, 261f\n"
-      "ld1 { v8.s }[2], [x28], #0x4\n"
-      "mov x19, #0xc\n"
-      "ld1 { v12.s }[2], [x24], #0x4\n"
-      "ld1 { v16.s }[2], [x23], #0x4\n"
-      "ld1 { v20.s }[2], [x22], #0x4\n"
-      "ld1 { v24.s }[2], [x21], #0x4\n"
-      "ld1 { v28.s }[2], [x20], #0x4\n"
+      "ld1 { v8.s }[2], [x9], #0x4\n"
+      "ld1 { v12.s }[2], [x25], #0x4\n"
+      "mov x20, #0xc\n"
+      "ld1 { v16.s }[2], [x24], #0x4\n"
+      "ld1 { v20.s }[2], [x23], #0x4\n"
+      "ld1 { v24.s }[2], [x22], #0x4\n"
+      "ld1 { v28.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v8.h }[6], [x28]\n"
-      "ld1 { v12.h }[6], [x24]\n"
-      "ld1 { v16.h }[6], [x23]\n"
-      "ld1 { v20.h }[6], [x22]\n"
-      "ld1 { v24.h }[6], [x21]\n"
-      "ld1 { v28.h }[6], [x20]\n"
+      "ld1 { v8.h }[6], [x9]\n"
+      "ld1 { v12.h }[6], [x25]\n"
+      "ld1 { v16.h }[6], [x24]\n"
+      "ld1 { v20.h }[6], [x23]\n"
+      "ld1 { v24.h }[6], [x22]\n"
+      "ld1 { v28.h }[6], [x21]\n"
       "b 264f\n"
       "261:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v8.h }[4], [x28]\n"
-      "ld1 { v12.h }[4], [x24]\n"
-      "ld1 { v16.h }[4], [x23]\n"
-      "ld1 { v20.h }[4], [x22]\n"
-      "ld1 { v24.h }[4], [x21]\n"
-      "ld1 { v28.h }[4], [x20]\n"
+      "ld1 { v8.h }[4], [x9]\n"
+      "ld1 { v12.h }[4], [x25]\n"
+      "ld1 { v16.h }[4], [x24]\n"
+      "ld1 { v20.h }[4], [x23]\n"
+      "ld1 { v24.h }[4], [x22]\n"
+      "ld1 { v28.h }[4], [x21]\n"
       "b 264f\n"
       "262:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x11, #1, 263f\n"
-      "ldr s8, [x28], #0x4\n"
-      "ldr s12, [x24], #0x4\n"
-      "mov x19, #0x4\n"
-      "ldr s16, [x23], #0x4\n"
-      "ldr s20, [x22], #0x4\n"
-      "ldr s24, [x21], #0x4\n"
-      "ldr s28, [x20], #0x4\n"
+      "ldr s8, [x9], #0x4\n"
+      "ldr s12, [x25], #0x4\n"
+      "mov x20, #0x4\n"
+      "ldr s16, [x24], #0x4\n"
+      "ldr s20, [x23], #0x4\n"
+      "ldr s24, [x22], #0x4\n"
+      "ldr s28, [x21], #0x4\n"
       "tbz x11, #0, 264f\n"
-      "ld1 { v8.h }[2], [x28]\n"
-      "ld1 { v12.h }[2], [x24]\n"
-      "ld1 { v16.h }[2], [x23]\n"
-      "ld1 { v20.h }[2], [x22]\n"
-      "ld1 { v24.h }[2], [x21]\n"
-      "ld1 { v28.h }[2], [x20]\n"
+      "ld1 { v8.h }[2], [x9]\n"
+      "ld1 { v12.h }[2], [x25]\n"
+      "ld1 { v16.h }[2], [x24]\n"
+      "ld1 { v20.h }[2], [x23]\n"
+      "ld1 { v24.h }[2], [x22]\n"
+      "ld1 { v28.h }[2], [x21]\n"
       "b 264f\n"
       "263:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr h8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr h12, [x24, #0x0]\n"
-      "ldr h16, [x23, #0x0]\n"
-      "ldr h20, [x22, #0x0]\n"
-      "ldr h24, [x21, #0x0]\n"
-      "ldr h28, [x20, #0x0]\n"
+      "ldr h8, [x9, #0x0]\n"
+      "ldr h12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr h16, [x24, #0x0]\n"
+      "ldr h20, [x23, #0x0]\n"
+      "ldr h24, [x22, #0x0]\n"
+      "ldr h28, [x21, #0x0]\n"
       "264:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 267f\n"
       "265:"  // Height 6: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "b 267f\n"
       "266:"  // Height 6: no accumulate
       "movi v8.16b, #0x0\n"
@@ -4382,82 +4382,82 @@ void a64_hybrid_fp16_mla_6x32 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "267:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "268:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 269f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 270f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 270f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 270f\n"
       "269:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "270:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "blt 273f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x10\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x10\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 272f\n"
       "271:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v9.8h, v7.8h, v0.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x10\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
+      "cmp x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -4659,51 +4659,51 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v30.8h, v6.8h, v5.h[7]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.8h, v7.8h, v0.h[7]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.8h, v7.8h, v1.h[7]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.8h, v7.8h, v2.h[7]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.8h, v7.8h, v3.h[7]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.8h, v7.8h, v4.h[7]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 271b\n"
       "272:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.8h, v6.8h, v0.h[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x8\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v24.8h, v6.8h, v4.h[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       "fmla v28.8h, v6.8h, v5.h[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
-      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x22, x22, #0x10\n"
+      "fmla v9.8h, v7.8h, v0.h[0]\n"
       "fmla v13.8h, v7.8h, v1.h[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x8\n"
       "fmla v17.8h, v7.8h, v2.h[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v21.8h, v7.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v25.8h, v7.8h, v4.h[0]\n"
       "fmla v29.8h, v7.8h, v5.h[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v10.8h, v6.8h, v0.h[0]\n"
       "fmla v14.8h, v6.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v18.8h, v6.8h, v2.h[0]\n"
       "fmla v22.8h, v6.8h, v3.h[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.8h, v6.8h, v4.h[0]\n"
       "fmla v30.8h, v6.8h, v5.h[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -4910,18 +4910,18 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v27.8h, v7.8h, v4.h[7]\n"
       "fmla v31.8h, v7.8h, v5.h[7]\n"
       "273:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 275f\n"
+      "cbz x27, 275f\n"
       "274:"  // Height 6: Multiply loop: Odd block loop
-      "ldr h0, [x25], #0x2\n"
-      "sub x26, x26, #0x1\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "sub x27, x27, #0x1\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.8h, v6.8h, v0.h[0]\n"
       "fmla v12.8h, v6.8h, v1.h[0]\n"
       "fmla v16.8h, v6.8h, v2.h[0]\n"
       "fmla v20.8h, v6.8h, v3.h[0]\n"
@@ -4948,379 +4948,379 @@ void a64_hybrid_fp16_mla_6x32 (
       "fmla v23.8h, v7.8h, v3.h[0]\n"
       "fmla v27.8h, v7.8h, v4.h[0]\n"
       "fmla v31.8h, v7.8h, v5.h[0]\n"
-      "cbnz x26, 274b\n"
+      "cbnz x27, 274b\n"
       "275:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 268b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #1\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #1\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #1\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbz %x[flags], #1, 276f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.8h }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.8h }, [x19]\n"
-      "fmin v8.8h, v8.8h, v0.8h\n"
-      "fmin v9.8h, v9.8h, v0.8h\n"
-      "fmin v10.8h, v10.8h, v0.8h\n"
-      "fmin v11.8h, v11.8h, v0.8h\n"
-      "fmax v8.8h, v8.8h, v1.8h\n"
-      "fmax v9.8h, v9.8h, v1.8h\n"
-      "fmax v10.8h, v10.8h, v1.8h\n"
-      "fmax v11.8h, v11.8h, v1.8h\n"
-      "fmin v12.8h, v12.8h, v0.8h\n"
-      "fmin v13.8h, v13.8h, v0.8h\n"
-      "fmin v14.8h, v14.8h, v0.8h\n"
-      "fmax v12.8h, v12.8h, v1.8h\n"
-      "fmax v13.8h, v13.8h, v1.8h\n"
-      "fmax v14.8h, v14.8h, v1.8h\n"
-      "fmin v15.8h, v15.8h, v0.8h\n"
-      "fmin v16.8h, v16.8h, v0.8h\n"
-      "fmin v17.8h, v17.8h, v0.8h\n"
-      "fmax v15.8h, v15.8h, v1.8h\n"
-      "fmax v16.8h, v16.8h, v1.8h\n"
-      "fmax v17.8h, v17.8h, v1.8h\n"
-      "fmin v18.8h, v18.8h, v0.8h\n"
-      "fmin v19.8h, v19.8h, v0.8h\n"
-      "fmin v20.8h, v20.8h, v0.8h\n"
-      "fmax v18.8h, v18.8h, v1.8h\n"
-      "fmax v19.8h, v19.8h, v1.8h\n"
-      "fmax v20.8h, v20.8h, v1.8h\n"
-      "fmin v21.8h, v21.8h, v0.8h\n"
-      "fmin v22.8h, v22.8h, v0.8h\n"
-      "fmin v23.8h, v23.8h, v0.8h\n"
-      "fmax v21.8h, v21.8h, v1.8h\n"
-      "fmax v22.8h, v22.8h, v1.8h\n"
-      "fmax v23.8h, v23.8h, v1.8h\n"
-      "fmin v24.8h, v24.8h, v0.8h\n"
-      "fmin v25.8h, v25.8h, v0.8h\n"
-      "fmin v26.8h, v26.8h, v0.8h\n"
-      "fmax v24.8h, v24.8h, v1.8h\n"
-      "fmax v25.8h, v25.8h, v1.8h\n"
-      "fmax v26.8h, v26.8h, v1.8h\n"
-      "fmin v27.8h, v27.8h, v0.8h\n"
-      "fmin v28.8h, v28.8h, v0.8h\n"
-      "fmin v29.8h, v29.8h, v0.8h\n"
-      "fmax v27.8h, v27.8h, v1.8h\n"
-      "fmax v28.8h, v28.8h, v1.8h\n"
-      "fmax v29.8h, v29.8h, v1.8h\n"
-      "fmin v30.8h, v30.8h, v0.8h\n"
-      "fmin v31.8h, v31.8h, v0.8h\n"
-      "fmax v30.8h, v30.8h, v1.8h\n"
-      "fmax v31.8h, v31.8h, v1.8h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.8h }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.8h }, [x20]\n"
+      "fmin v8.8h, v8.8h, v1.8h\n"
+      "fmin v9.8h, v9.8h, v1.8h\n"
+      "fmin v10.8h, v10.8h, v1.8h\n"
+      "fmin v11.8h, v11.8h, v1.8h\n"
+      "fmin v12.8h, v12.8h, v1.8h\n"
+      "fmin v13.8h, v13.8h, v1.8h\n"
+      "fmin v14.8h, v14.8h, v1.8h\n"
+      "fmin v15.8h, v15.8h, v1.8h\n"
+      "fmin v16.8h, v16.8h, v1.8h\n"
+      "fmin v17.8h, v17.8h, v1.8h\n"
+      "fmin v18.8h, v18.8h, v1.8h\n"
+      "fmin v19.8h, v19.8h, v1.8h\n"
+      "fmin v20.8h, v20.8h, v1.8h\n"
+      "fmin v21.8h, v21.8h, v1.8h\n"
+      "fmin v22.8h, v22.8h, v1.8h\n"
+      "fmin v23.8h, v23.8h, v1.8h\n"
+      "fmin v24.8h, v24.8h, v1.8h\n"
+      "fmin v25.8h, v25.8h, v1.8h\n"
+      "fmin v26.8h, v26.8h, v1.8h\n"
+      "fmin v27.8h, v27.8h, v1.8h\n"
+      "fmin v28.8h, v28.8h, v1.8h\n"
+      "fmin v29.8h, v29.8h, v1.8h\n"
+      "fmin v30.8h, v30.8h, v1.8h\n"
+      "fmin v31.8h, v31.8h, v1.8h\n"
+      "fmax v8.8h, v8.8h, v0.8h\n"
+      "fmax v9.8h, v9.8h, v0.8h\n"
+      "fmax v10.8h, v10.8h, v0.8h\n"
+      "fmax v11.8h, v11.8h, v0.8h\n"
+      "fmax v12.8h, v12.8h, v0.8h\n"
+      "fmax v13.8h, v13.8h, v0.8h\n"
+      "fmax v14.8h, v14.8h, v0.8h\n"
+      "fmax v15.8h, v15.8h, v0.8h\n"
+      "fmax v16.8h, v16.8h, v0.8h\n"
+      "fmax v17.8h, v17.8h, v0.8h\n"
+      "fmax v18.8h, v18.8h, v0.8h\n"
+      "fmax v19.8h, v19.8h, v0.8h\n"
+      "fmax v20.8h, v20.8h, v0.8h\n"
+      "fmax v21.8h, v21.8h, v0.8h\n"
+      "fmax v22.8h, v22.8h, v0.8h\n"
+      "fmax v23.8h, v23.8h, v0.8h\n"
+      "fmax v24.8h, v24.8h, v0.8h\n"
+      "fmax v25.8h, v25.8h, v0.8h\n"
+      "fmax v26.8h, v26.8h, v0.8h\n"
+      "fmax v27.8h, v27.8h, v0.8h\n"
+      "fmax v28.8h, v28.8h, v0.8h\n"
+      "fmax v29.8h, v29.8h, v0.8h\n"
+      "fmax v30.8h, v30.8h, v0.8h\n"
+      "fmax v31.8h, v31.8h, v0.8h\n"
       "276:"  // Height 6: No activation
       "cmp x11, #0x20\n"
       "bge 293f\n"
       "tbz x11, #4, 284f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v9.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v13.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v17.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v21.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v25.8h }, [x21], #0x10\n"
-      "st1 { v28.8h }, [x20], #0x10\n"
-      "st1 { v29.8h }, [x20], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v9.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v13.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v17.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v21.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v25.8h }, [x22], #0x10\n"
+      "st1 { v28.8h }, [x21], #0x10\n"
+      "st1 { v29.8h }, [x21], #0x10\n"
       "tbz x11, #3, 280f\n"
-      "st1 { v10.8h }, [x28], #0x10\n"
-      "st1 { v14.8h }, [x24], #0x10\n"
-      "st1 { v18.8h }, [x23], #0x10\n"
-      "st1 { v22.8h }, [x22], #0x10\n"
-      "st1 { v26.8h }, [x21], #0x10\n"
-      "st1 { v30.8h }, [x20], #0x10\n"
+      "st1 { v10.8h }, [x9], #0x10\n"
+      "st1 { v14.8h }, [x25], #0x10\n"
+      "st1 { v18.8h }, [x24], #0x10\n"
+      "st1 { v22.8h }, [x23], #0x10\n"
+      "st1 { v26.8h }, [x22], #0x10\n"
+      "st1 { v30.8h }, [x21], #0x10\n"
       "tbz x11, #2, 278f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
       "tbz x11, #1, 277f\n"
-      "st1 { v11.s }[2], [x28], #0x4\n"
-      "st1 { v15.s }[2], [x24], #0x4\n"
-      "st1 { v19.s }[2], [x23], #0x4\n"
-      "st1 { v23.s }[2], [x22], #0x4\n"
-      "st1 { v27.s }[2], [x21], #0x4\n"
-      "st1 { v31.s }[2], [x20], #0x4\n"
+      "st1 { v11.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x25], #0x4\n"
+      "st1 { v19.s }[2], [x24], #0x4\n"
+      "st1 { v23.s }[2], [x23], #0x4\n"
+      "st1 { v27.s }[2], [x22], #0x4\n"
+      "st1 { v31.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v11.h }[6], [x28]\n"
-      "st1 { v15.h }[6], [x24]\n"
-      "st1 { v19.h }[6], [x23]\n"
-      "st1 { v23.h }[6], [x22]\n"
-      "st1 { v27.h }[6], [x21]\n"
-      "st1 { v31.h }[6], [x20]\n"
+      "st1 { v11.h }[6], [x9]\n"
+      "st1 { v15.h }[6], [x25]\n"
+      "st1 { v19.h }[6], [x24]\n"
+      "st1 { v23.h }[6], [x23]\n"
+      "st1 { v27.h }[6], [x22]\n"
+      "st1 { v31.h }[6], [x21]\n"
       "b 292f\n"
       "277:"  // Height 6: Partial direct writeback: partial_1_28
       "tbz x11, #0, 292f\n"
-      "st1 { v11.h }[4], [x28]\n"
-      "st1 { v15.h }[4], [x24]\n"
-      "st1 { v19.h }[4], [x23]\n"
-      "st1 { v23.h }[4], [x22]\n"
-      "st1 { v27.h }[4], [x21]\n"
-      "st1 { v31.h }[4], [x20]\n"
+      "st1 { v11.h }[4], [x9]\n"
+      "st1 { v15.h }[4], [x25]\n"
+      "st1 { v19.h }[4], [x24]\n"
+      "st1 { v23.h }[4], [x23]\n"
+      "st1 { v27.h }[4], [x22]\n"
+      "st1 { v31.h }[4], [x21]\n"
       "b 292f\n"
       "278:"  // Height 6: Partial direct writeback: partial_2_24
       "tbz x11, #1, 279f\n"
-      "str s11, [x28], #0x4\n"
-      "str s15, [x24], #0x4\n"
-      "str s19, [x23], #0x4\n"
-      "str s23, [x22], #0x4\n"
-      "str s27, [x21], #0x4\n"
-      "str s31, [x20], #0x4\n"
+      "str s11, [x9], #0x4\n"
+      "str s15, [x25], #0x4\n"
+      "str s19, [x24], #0x4\n"
+      "str s23, [x23], #0x4\n"
+      "str s27, [x22], #0x4\n"
+      "str s31, [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v11.h }[2], [x28]\n"
-      "st1 { v15.h }[2], [x24]\n"
-      "st1 { v19.h }[2], [x23]\n"
-      "st1 { v23.h }[2], [x22]\n"
-      "st1 { v27.h }[2], [x21]\n"
-      "st1 { v31.h }[2], [x20]\n"
+      "st1 { v11.h }[2], [x9]\n"
+      "st1 { v15.h }[2], [x25]\n"
+      "st1 { v19.h }[2], [x24]\n"
+      "st1 { v23.h }[2], [x23]\n"
+      "st1 { v27.h }[2], [x22]\n"
+      "st1 { v31.h }[2], [x21]\n"
       "b 292f\n"
       "279:"  // Height 6: Partial direct writeback: partial_1_24
       "tbz x11, #0, 292f\n"
-      "str h11, [x28, #0x0]\n"
-      "str h15, [x24, #0x0]\n"
-      "str h19, [x23, #0x0]\n"
-      "str h23, [x22, #0x0]\n"
-      "str h27, [x21, #0x0]\n"
-      "str h31, [x20, #0x0]\n"
+      "str h11, [x9, #0x0]\n"
+      "str h15, [x25, #0x0]\n"
+      "str h19, [x24, #0x0]\n"
+      "str h23, [x23, #0x0]\n"
+      "str h27, [x22, #0x0]\n"
+      "str h31, [x21, #0x0]\n"
       "b 292f\n"
       "280:"  // Height 6: Partial direct writeback: partial_4_16
       "tbz x11, #2, 282f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
       "tbz x11, #1, 281f\n"
-      "st1 { v10.s }[2], [x28], #0x4\n"
-      "st1 { v14.s }[2], [x24], #0x4\n"
-      "st1 { v18.s }[2], [x23], #0x4\n"
-      "st1 { v22.s }[2], [x22], #0x4\n"
-      "st1 { v26.s }[2], [x21], #0x4\n"
-      "st1 { v30.s }[2], [x20], #0x4\n"
+      "st1 { v10.s }[2], [x9], #0x4\n"
+      "st1 { v14.s }[2], [x25], #0x4\n"
+      "st1 { v18.s }[2], [x24], #0x4\n"
+      "st1 { v22.s }[2], [x23], #0x4\n"
+      "st1 { v26.s }[2], [x22], #0x4\n"
+      "st1 { v30.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v10.h }[6], [x28]\n"
-      "st1 { v14.h }[6], [x24]\n"
-      "st1 { v18.h }[6], [x23]\n"
-      "st1 { v22.h }[6], [x22]\n"
-      "st1 { v26.h }[6], [x21]\n"
-      "st1 { v30.h }[6], [x20]\n"
+      "st1 { v10.h }[6], [x9]\n"
+      "st1 { v14.h }[6], [x25]\n"
+      "st1 { v18.h }[6], [x24]\n"
+      "st1 { v22.h }[6], [x23]\n"
+      "st1 { v26.h }[6], [x22]\n"
+      "st1 { v30.h }[6], [x21]\n"
       "b 292f\n"
       "281:"  // Height 6: Partial direct writeback: partial_1_20
       "tbz x11, #0, 292f\n"
-      "st1 { v10.h }[4], [x28]\n"
-      "st1 { v14.h }[4], [x24]\n"
-      "st1 { v18.h }[4], [x23]\n"
-      "st1 { v22.h }[4], [x22]\n"
-      "st1 { v26.h }[4], [x21]\n"
-      "st1 { v30.h }[4], [x20]\n"
+      "st1 { v10.h }[4], [x9]\n"
+      "st1 { v14.h }[4], [x25]\n"
+      "st1 { v18.h }[4], [x24]\n"
+      "st1 { v22.h }[4], [x23]\n"
+      "st1 { v26.h }[4], [x22]\n"
+      "st1 { v30.h }[4], [x21]\n"
       "b 292f\n"
       "282:"  // Height 6: Partial direct writeback: partial_2_16
       "tbz x11, #1, 283f\n"
-      "str s10, [x28], #0x4\n"
-      "str s14, [x24], #0x4\n"
-      "str s18, [x23], #0x4\n"
-      "str s22, [x22], #0x4\n"
-      "str s26, [x21], #0x4\n"
-      "str s30, [x20], #0x4\n"
+      "str s10, [x9], #0x4\n"
+      "str s14, [x25], #0x4\n"
+      "str s18, [x24], #0x4\n"
+      "str s22, [x23], #0x4\n"
+      "str s26, [x22], #0x4\n"
+      "str s30, [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v10.h }[2], [x28]\n"
-      "st1 { v14.h }[2], [x24]\n"
-      "st1 { v18.h }[2], [x23]\n"
-      "st1 { v22.h }[2], [x22]\n"
-      "st1 { v26.h }[2], [x21]\n"
-      "st1 { v30.h }[2], [x20]\n"
+      "st1 { v10.h }[2], [x9]\n"
+      "st1 { v14.h }[2], [x25]\n"
+      "st1 { v18.h }[2], [x24]\n"
+      "st1 { v22.h }[2], [x23]\n"
+      "st1 { v26.h }[2], [x22]\n"
+      "st1 { v30.h }[2], [x21]\n"
       "b 292f\n"
       "283:"  // Height 6: Partial direct writeback: partial_1_16
       "tbz x11, #0, 292f\n"
-      "str h10, [x28, #0x0]\n"
-      "str h14, [x24, #0x0]\n"
-      "str h18, [x23, #0x0]\n"
-      "str h22, [x22, #0x0]\n"
-      "str h26, [x21, #0x0]\n"
-      "str h30, [x20, #0x0]\n"
+      "str h10, [x9, #0x0]\n"
+      "str h14, [x25, #0x0]\n"
+      "str h18, [x24, #0x0]\n"
+      "str h22, [x23, #0x0]\n"
+      "str h26, [x22, #0x0]\n"
+      "str h30, [x21, #0x0]\n"
       "b 292f\n"
       "284:"  // Height 6: Partial direct writeback: partial_8_0
       "tbz x11, #3, 288f\n"
-      "st1 { v8.8h }, [x28], #0x10\n"
-      "st1 { v12.8h }, [x24], #0x10\n"
-      "st1 { v16.8h }, [x23], #0x10\n"
-      "st1 { v20.8h }, [x22], #0x10\n"
-      "st1 { v24.8h }, [x21], #0x10\n"
-      "st1 { v28.8h }, [x20], #0x10\n"
+      "st1 { v8.8h }, [x9], #0x10\n"
+      "st1 { v12.8h }, [x25], #0x10\n"
+      "st1 { v16.8h }, [x24], #0x10\n"
+      "st1 { v20.8h }, [x23], #0x10\n"
+      "st1 { v24.8h }, [x22], #0x10\n"
+      "st1 { v28.8h }, [x21], #0x10\n"
       "tbz x11, #2, 286f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
       "tbz x11, #1, 285f\n"
-      "st1 { v9.s }[2], [x28], #0x4\n"
-      "st1 { v13.s }[2], [x24], #0x4\n"
-      "st1 { v17.s }[2], [x23], #0x4\n"
-      "st1 { v21.s }[2], [x22], #0x4\n"
-      "st1 { v25.s }[2], [x21], #0x4\n"
-      "st1 { v29.s }[2], [x20], #0x4\n"
+      "st1 { v9.s }[2], [x9], #0x4\n"
+      "st1 { v13.s }[2], [x25], #0x4\n"
+      "st1 { v17.s }[2], [x24], #0x4\n"
+      "st1 { v21.s }[2], [x23], #0x4\n"
+      "st1 { v25.s }[2], [x22], #0x4\n"
+      "st1 { v29.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v9.h }[6], [x28]\n"
-      "st1 { v13.h }[6], [x24]\n"
-      "st1 { v17.h }[6], [x23]\n"
-      "st1 { v21.h }[6], [x22]\n"
-      "st1 { v25.h }[6], [x21]\n"
-      "st1 { v29.h }[6], [x20]\n"
+      "st1 { v9.h }[6], [x9]\n"
+      "st1 { v13.h }[6], [x25]\n"
+      "st1 { v17.h }[6], [x24]\n"
+      "st1 { v21.h }[6], [x23]\n"
+      "st1 { v25.h }[6], [x22]\n"
+      "st1 { v29.h }[6], [x21]\n"
       "b 292f\n"
       "285:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x11, #0, 292f\n"
-      "st1 { v9.h }[4], [x28]\n"
-      "st1 { v13.h }[4], [x24]\n"
-      "st1 { v17.h }[4], [x23]\n"
-      "st1 { v21.h }[4], [x22]\n"
-      "st1 { v25.h }[4], [x21]\n"
-      "st1 { v29.h }[4], [x20]\n"
+      "st1 { v9.h }[4], [x9]\n"
+      "st1 { v13.h }[4], [x25]\n"
+      "st1 { v17.h }[4], [x24]\n"
+      "st1 { v21.h }[4], [x23]\n"
+      "st1 { v25.h }[4], [x22]\n"
+      "st1 { v29.h }[4], [x21]\n"
       "b 292f\n"
       "286:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x11, #1, 287f\n"
-      "str s9, [x28], #0x4\n"
-      "str s13, [x24], #0x4\n"
-      "str s17, [x23], #0x4\n"
-      "str s21, [x22], #0x4\n"
-      "str s25, [x21], #0x4\n"
-      "str s29, [x20], #0x4\n"
+      "str s9, [x9], #0x4\n"
+      "str s13, [x25], #0x4\n"
+      "str s17, [x24], #0x4\n"
+      "str s21, [x23], #0x4\n"
+      "str s25, [x22], #0x4\n"
+      "str s29, [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v9.h }[2], [x28]\n"
-      "st1 { v13.h }[2], [x24]\n"
-      "st1 { v17.h }[2], [x23]\n"
-      "st1 { v21.h }[2], [x22]\n"
-      "st1 { v25.h }[2], [x21]\n"
-      "st1 { v29.h }[2], [x20]\n"
+      "st1 { v9.h }[2], [x9]\n"
+      "st1 { v13.h }[2], [x25]\n"
+      "st1 { v17.h }[2], [x24]\n"
+      "st1 { v21.h }[2], [x23]\n"
+      "st1 { v25.h }[2], [x22]\n"
+      "st1 { v29.h }[2], [x21]\n"
       "b 292f\n"
       "287:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x11, #0, 292f\n"
-      "str h9, [x28, #0x0]\n"
-      "str h13, [x24, #0x0]\n"
-      "str h17, [x23, #0x0]\n"
-      "str h21, [x22, #0x0]\n"
-      "str h25, [x21, #0x0]\n"
-      "str h29, [x20, #0x0]\n"
+      "str h9, [x9, #0x0]\n"
+      "str h13, [x25, #0x0]\n"
+      "str h17, [x24, #0x0]\n"
+      "str h21, [x23, #0x0]\n"
+      "str h25, [x22, #0x0]\n"
+      "str h29, [x21, #0x0]\n"
       "b 292f\n"
       "288:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x11, #2, 290f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
       "tbz x11, #1, 289f\n"
-      "st1 { v8.s }[2], [x28], #0x4\n"
-      "st1 { v12.s }[2], [x24], #0x4\n"
-      "st1 { v16.s }[2], [x23], #0x4\n"
-      "st1 { v20.s }[2], [x22], #0x4\n"
-      "st1 { v24.s }[2], [x21], #0x4\n"
-      "st1 { v28.s }[2], [x20], #0x4\n"
+      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v12.s }[2], [x25], #0x4\n"
+      "st1 { v16.s }[2], [x24], #0x4\n"
+      "st1 { v20.s }[2], [x23], #0x4\n"
+      "st1 { v24.s }[2], [x22], #0x4\n"
+      "st1 { v28.s }[2], [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v8.h }[6], [x28]\n"
-      "st1 { v12.h }[6], [x24]\n"
-      "st1 { v16.h }[6], [x23]\n"
-      "st1 { v20.h }[6], [x22]\n"
-      "st1 { v24.h }[6], [x21]\n"
-      "st1 { v28.h }[6], [x20]\n"
+      "st1 { v8.h }[6], [x9]\n"
+      "st1 { v12.h }[6], [x25]\n"
+      "st1 { v16.h }[6], [x24]\n"
+      "st1 { v20.h }[6], [x23]\n"
+      "st1 { v24.h }[6], [x22]\n"
+      "st1 { v28.h }[6], [x21]\n"
       "b 292f\n"
       "289:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x11, #0, 292f\n"
-      "st1 { v8.h }[4], [x28]\n"
-      "st1 { v12.h }[4], [x24]\n"
-      "st1 { v16.h }[4], [x23]\n"
-      "st1 { v20.h }[4], [x22]\n"
-      "st1 { v24.h }[4], [x21]\n"
-      "st1 { v28.h }[4], [x20]\n"
+      "st1 { v8.h }[4], [x9]\n"
+      "st1 { v12.h }[4], [x25]\n"
+      "st1 { v16.h }[4], [x24]\n"
+      "st1 { v20.h }[4], [x23]\n"
+      "st1 { v24.h }[4], [x22]\n"
+      "st1 { v28.h }[4], [x21]\n"
       "b 292f\n"
       "290:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x11, #1, 291f\n"
-      "str s8, [x28], #0x4\n"
-      "str s12, [x24], #0x4\n"
-      "str s16, [x23], #0x4\n"
-      "str s20, [x22], #0x4\n"
-      "str s24, [x21], #0x4\n"
-      "str s28, [x20], #0x4\n"
+      "str s8, [x9], #0x4\n"
+      "str s12, [x25], #0x4\n"
+      "str s16, [x24], #0x4\n"
+      "str s20, [x23], #0x4\n"
+      "str s24, [x22], #0x4\n"
+      "str s28, [x21], #0x4\n"
       "tbz x11, #0, 292f\n"
-      "st1 { v8.h }[2], [x28]\n"
-      "st1 { v12.h }[2], [x24]\n"
-      "st1 { v16.h }[2], [x23]\n"
-      "st1 { v20.h }[2], [x22]\n"
-      "st1 { v24.h }[2], [x21]\n"
-      "st1 { v28.h }[2], [x20]\n"
+      "st1 { v8.h }[2], [x9]\n"
+      "st1 { v12.h }[2], [x25]\n"
+      "st1 { v16.h }[2], [x24]\n"
+      "st1 { v20.h }[2], [x23]\n"
+      "st1 { v24.h }[2], [x22]\n"
+      "st1 { v28.h }[2], [x21]\n"
       "b 292f\n"
       "291:"  // Height 6: Partial direct writeback: partial_1_0
-      "str h8, [x28, #0x0]\n"
-      "str h12, [x24, #0x0]\n"
-      "str h16, [x23, #0x0]\n"
-      "str h20, [x22, #0x0]\n"
-      "str h24, [x21, #0x0]\n"
-      "str h28, [x20, #0x0]\n"
+      "str h8, [x9, #0x0]\n"
+      "str h12, [x25, #0x0]\n"
+      "str h16, [x24, #0x0]\n"
+      "str h20, [x23, #0x0]\n"
+      "str h24, [x22, #0x0]\n"
+      "str h28, [x21, #0x0]\n"
       "292:"  // Height 6: Partial direct writeback: Done
       "b 294f\n"
       "293:"  // Height 6: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
-      "str q28, [x20, #0x0]\n"
-      "str q29, [x20, #0x10]\n"
-      "str q30, [x20, #0x20]\n"
-      "str q31, [x20, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
+      "str q28, [x21, #0x0]\n"
+      "str q29, [x21, #0x10]\n"
+      "str q30, [x21, #0x20]\n"
+      "str q31, [x21, #0x30]\n"
       "294:"  // Height 6: Writeback done
       "subs x11, x11, #0x20\n"
       "bgt 247b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 296f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 295f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "295:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "296:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp
index 1fbc9232f0..700d803f82 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -99,117 +99,117 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 83f\n"
       "beq 42f\n"
+      "mov x17, %x[bias]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x13, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x14, 3f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
-      "ldr q11, [x14, #0x30]\n"
-      "ldr q12, [x14, #0x40]\n"
-      "ldr q13, [x14, #0x50]\n"
-      "add x14, x14, #0x60\n"
+      "cbz x17, 3f\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x17, #0x40]\n"
+      "ldr q13, [x17, #0x50]\n"
+      "add x17, x17, #0x60\n"
       "b 18f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 17f\n"
       "cmp x16, #0x18\n"
       "bge 16f\n"
       "tbz x16, #4, 7f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
-      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
+      "ld1 { v11.4s }, [x14], #0x10\n"
       "tbz x16, #2, 5f\n"
-      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x14], #0x10\n"
       "tbz x16, #1, 4f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x13], #0x8\n"
+      "ldr d13, [x14], #0x8\n"
+      "mov x20, #0x58\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x14]\n"
       "b 15f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x16, #0, 15f\n"
-      "ldr s13, [x13, #0x0]\n"
+      "ldr s13, [x14, #0x0]\n"
       "b 15f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_16
       "tbz x16, #1, 6f\n"
-      "ldr d12, [x13], #0x8\n"
-      "mov x19, #0x48\n"
+      "ldr d12, [x14], #0x8\n"
+      "mov x20, #0x48\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x14]\n"
       "b 15f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x16, #0, 15f\n"
-      "ldr s12, [x13, #0x0]\n"
+      "ldr s12, [x14, #0x0]\n"
       "b 15f\n"
       "7:"  // Height 1: Partial accumulate: partial_8_0
       "tbz x16, #3, 11f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "tbz x16, #2, 9f\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "tbz x16, #1, 8f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x13], #0x8\n"
+      "ldr d11, [x14], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v11.s }[2], [x14]\n"
       "b 15f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x16, #0, 15f\n"
-      "ldr s11, [x13, #0x0]\n"
+      "ldr s11, [x14, #0x0]\n"
       "b 15f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x16, #1, 10f\n"
-      "ldr d10, [x13], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x14], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v10.s }[2], [x14]\n"
       "b 15f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x16, #0, 15f\n"
-      "ldr s10, [x13, #0x0]\n"
+      "ldr s10, [x14, #0x0]\n"
       "b 15f\n"
       "11:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x16, #2, 13f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "tbz x16, #1, 12f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x13], #0x8\n"
+      "ldr d9, [x14], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v9.s }[2], [x14]\n"
       "b 15f\n"
       "12:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x16, #0, 15f\n"
-      "ldr s9, [x13, #0x0]\n"
+      "ldr s9, [x14, #0x0]\n"
       "b 15f\n"
       "13:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x16, #1, 14f\n"
-      "ldr d8, [x13], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x14], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x16, #0, 15f\n"
-      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v8.s }[2], [x14]\n"
       "b 15f\n"
       "14:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x13, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x14, #0x0]\n"
+      "mov x20, #0x0\n"
       "15:"  // Height 1: Partial accumulate: Done
-      "sub x13, x13, x19\n"
+      "sub x14, x14, x20\n"
       "b 18f\n"
       "16:"  // Height 1: full accumulate
-      "ldr q8, [x13, #0x0]\n"
-      "ldr q9, [x13, #0x10]\n"
-      "ldr q10, [x13, #0x20]\n"
-      "ldr q11, [x13, #0x30]\n"
-      "ldr q12, [x13, #0x40]\n"
-      "ldr q13, [x13, #0x50]\n"
+      "ldr q8, [x14, #0x0]\n"
+      "ldr q9, [x14, #0x10]\n"
+      "ldr q10, [x14, #0x20]\n"
+      "ldr q11, [x14, #0x30]\n"
+      "ldr q12, [x14, #0x40]\n"
+      "ldr q13, [x14, #0x50]\n"
       "b 18f\n"
       "17:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -219,463 +219,463 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "movi v12.16b, #0x0\n"
       "movi v13.16b, #0x0\n"
       "18:"  // Height 1: setup done
-      "mov x12, #0x0\n"
+      "mov x13, #0x0\n"
       "19:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr w12, [x20, x13, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x10, [x20, #0x0]\n"
-      "cbnz x12, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x10, x10, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x11, [x21, #0x0]\n"
+      "cbnz x13, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 1: setup direct input
-      "mov x10, %x[input_ptr]\n"
+      "mov x11, %x[input_ptr]\n"
       "21:"  // Height 1: input setup done
-      "cmp x11, #0x4\n"
+      "cmp x12, #0x4\n"
       "blt 24f\n"
-      "ldr q0, [x10, #0x0]\n"
+      "ldr q0, [x11, #0x0]\n"
+      "cmp x12, #0x8\n"
       "ldr q4, [x15, #0x0]\n"
-      "cmp x11, #0x8\n"
+      "ldr q5, [x15, #0x10]\n"
+      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x30]\n"
       "blt 23f\n"
       "22:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr d5, [x15, #0x10]\n"
-      "ldr x9, [x15, #0x18]\n"
-      "add x10, x10, #0x10\n"
-      "ldr d6, [x15, #0x20]\n"
-      "sub x11, x11, #0x4\n"
-      "ldr x28, [x15, #0x28]\n"
-      "cmp x11, #0x8\n"
-      "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x30]\n"
-      "ldr x27, [x15, #0x38]\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "mov v6.d[1], x28\n"
       "ldr d4, [x15, #0x40]\n"
+      "ldr x10, [x15, #0x48]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "ldr d5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "mov v7.d[1], x27\n"
-      "ldr x26, [x15, #0x48]\n"
+      "ldr d6, [x15, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "ldr d5, [x15, #0x50]\n"
+      "ldr d7, [x15, #0x70]\n"
+      "mov v4.d[1], x10\n"
       "ldr x9, [x15, #0x58]\n"
-      "mov v4.d[1], x26\n"
-      "ldr d6, [x15, #0x60]\n"
-      "ldr x28, [x15, #0x68]\n"
-      "fmla v12.4s, v4.4s, v0.s[0]\n"
       "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x70]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "ldr x28, [x15, #0x68]\n"
       "mov v6.d[1], x28\n"
       "ldr x27, [x15, #0x78]\n"
-      "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr d4, [x15, #0x80]\n"
-      "ldr x26, [x15, #0x88]\n"
       "mov v7.d[1], x27\n"
+      "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "ldr d4, [x15, #0x80]\n"
+      "ldr x10, [x15, #0x88]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "ldr d5, [x15, #0x90]\n"
-      "ldr x9, [x15, #0x98]\n"
-      "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr d6, [x15, #0xa0]\n"
-      "fmla v10.4s, v4.4s, v0.s[1]\n"
+      "fmla v9.4s, v7.4s, v0.s[1]\n"
+      "ldr d7, [x15, #0xb0]\n"
+      "mov v4.d[1], x10\n"
+      "ldr x9, [x15, #0x98]\n"
       "mov v5.d[1], x9\n"
       "ldr x28, [x15, #0xa8]\n"
-      "fmla v11.4s, v5.4s, v0.s[1]\n"
-      "ldr d7, [x15, #0xb0]\n"
-      "ldr x27, [x15, #0xb8]\n"
       "mov v6.d[1], x28\n"
-      "ldr d4, [x15, #0xc0]\n"
-      "ldr x26, [x15, #0xc8]\n"
-      "fmla v12.4s, v6.4s, v0.s[1]\n"
+      "ldr x27, [x15, #0xb8]\n"
       "mov v7.d[1], x27\n"
+      "fmla v10.4s, v4.4s, v0.s[1]\n"
+      "ldr d4, [x15, #0xc0]\n"
+      "ldr x10, [x15, #0xc8]\n"
+      "fmla v11.4s, v5.4s, v0.s[1]\n"
       "ldr d5, [x15, #0xd0]\n"
+      "fmla v12.4s, v6.4s, v0.s[1]\n"
+      "ldr d6, [x15, #0xe0]\n"
       "fmla v13.4s, v7.4s, v0.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "ldr d7, [x15, #0xf0]\n"
+      "mov v4.d[1], x10\n"
       "ldr x9, [x15, #0xd8]\n"
-      "fmla v8.4s, v4.4s, v0.s[2]\n"
-      "ldr d6, [x15, #0xe0]\n"
-      "ldr x28, [x15, #0xe8]\n"
       "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0xf0]\n"
-      "ldr x27, [x15, #0xf8]\n"
-      "fmla v9.4s, v5.4s, v0.s[2]\n"
+      "ldr x28, [x15, #0xe8]\n"
       "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0xf8]\n"
+      "mov v7.d[1], x27\n"
+      "fmla v8.4s, v4.4s, v0.s[2]\n"
       "ldr d4, [x15, #0x100]\n"
+      "ldr x10, [x15, #0x108]\n"
+      "fmla v9.4s, v5.4s, v0.s[2]\n"
+      "ldr d5, [x15, #0x110]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "mov v7.d[1], x27\n"
-      "ldr x26, [x15, #0x108]\n"
+      "ldr d6, [x15, #0x120]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr d5, [x15, #0x110]\n"
+      "ldr d7, [x15, #0x130]\n"
+      "mov v4.d[1], x10\n"
       "ldr x9, [x15, #0x118]\n"
-      "mov v4.d[1], x26\n"
-      "ldr d6, [x15, #0x120]\n"
-      "ldr x28, [x15, #0x128]\n"
-      "fmla v12.4s, v4.4s, v0.s[2]\n"
       "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x130]\n"
-      "fmla v13.4s, v5.4s, v0.s[2]\n"
+      "ldr x28, [x15, #0x128]\n"
       "mov v6.d[1], x28\n"
       "ldr x27, [x15, #0x138]\n"
-      "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr d4, [x15, #0x140]\n"
-      "ldr x26, [x15, #0x148]\n"
       "mov v7.d[1], x27\n"
+      "fmla v12.4s, v4.4s, v0.s[2]\n"
+      "ldr d4, [x15, #0x140]\n"
+      "ldr x10, [x15, #0x148]\n"
+      "fmla v13.4s, v5.4s, v0.s[2]\n"
       "ldr d5, [x15, #0x150]\n"
-      "ldr x9, [x15, #0x158]\n"
-      "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "mov v4.d[1], x26\n"
+      "fmla v8.4s, v6.4s, v0.s[3]\n"
       "ldr d6, [x15, #0x160]\n"
-      "fmla v10.4s, v4.4s, v0.s[3]\n"
+      "fmla v9.4s, v7.4s, v0.s[3]\n"
+      "ldr d7, [x15, #0x170]\n"
+      "mov v4.d[1], x10\n"
+      "ldr x9, [x15, #0x158]\n"
       "mov v5.d[1], x9\n"
       "ldr x28, [x15, #0x168]\n"
-      "fmla v11.4s, v5.4s, v0.s[3]\n"
-      "ldr d7, [x15, #0x170]\n"
-      "ldr x27, [x15, #0x178]\n"
-      "add x15, x15, #0x180\n"
       "mov v6.d[1], x28\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
-      "ldr x25, [x10, #0x8]\n"
-      "fmla v12.4s, v6.4s, v0.s[3]\n"
+      "ldr x27, [x15, #0x178]\n"
       "mov v7.d[1], x27\n"
+      "add x11, x11, #0x10\n"
+      "add x15, x15, #0x180\n"
+      "fmla v10.4s, v4.4s, v0.s[3]\n"
       "ldr d4, [x15, #0x0]\n"
+      "ldr x10, [x15, #0x8]\n"
+      "fmla v11.4s, v5.4s, v0.s[3]\n"
+      "ldr d5, [x15, #0x10]\n"
+      "fmla v12.4s, v6.4s, v0.s[3]\n"
+      "ldr d6, [x15, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x10, #0x0]\n"
-      "ldr x26, [x15, #0x8]\n"
-      "mov v0.d[1], x25\n"
-      "mov v4.d[1], x26\n"
+      "ldr d0, [x11, #0x0]\n"
+      "sub x12, x12, #0x4\n"
+      "ldr d7, [x15, #0x30]\n"
+      "cmp x12, #0x8\n"
+      "ldr x9, [x15, #0x18]\n"
+      "mov v4.d[1], x10\n"
+      "ldr x28, [x15, #0x28]\n"
+      "mov v5.d[1], x9\n"
+      "ldr x26, [x11, #0x8]\n"
+      "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0x38]\n"
+      "mov v0.d[1], x26\n"
+      "mov v7.d[1], x27\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "bge 22b\n"
       "23:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x15, #0x10]\n"
-      "ldr q6, [x15, #0x20]\n"
-      "sub x11, x11, #0x4\n"
-      "ldr q7, [x15, #0x30]\n"
-      "add x10, x10, #0x10\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x15, #0x40]\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x15, #0x50]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x15, #0x60]\n"
-      "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x15, #0x70]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "fmla v12.4s, v4.4s, v0.s[0]\n"
       "ldr q4, [x15, #0x80]\n"
-      "fmla v8.4s, v6.4s, v0.s[1]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x15, #0x90]\n"
-      "fmla v9.4s, v7.4s, v0.s[1]\n"
+      "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x15, #0xa0]\n"
-      "fmla v10.4s, v4.4s, v0.s[1]\n"
+      "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x15, #0xb0]\n"
-      "fmla v11.4s, v5.4s, v0.s[1]\n"
+      "fmla v10.4s, v4.4s, v0.s[1]\n"
       "ldr q4, [x15, #0xc0]\n"
-      "fmla v12.4s, v6.4s, v0.s[1]\n"
+      "fmla v11.4s, v5.4s, v0.s[1]\n"
       "ldr q5, [x15, #0xd0]\n"
-      "fmla v13.4s, v7.4s, v0.s[1]\n"
+      "fmla v12.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x15, #0xe0]\n"
-      "fmla v8.4s, v4.4s, v0.s[2]\n"
+      "fmla v13.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x15, #0xf0]\n"
-      "fmla v9.4s, v5.4s, v0.s[2]\n"
+      "fmla v8.4s, v4.4s, v0.s[2]\n"
       "ldr q4, [x15, #0x100]\n"
-      "fmla v10.4s, v6.4s, v0.s[2]\n"
+      "fmla v9.4s, v5.4s, v0.s[2]\n"
       "ldr q5, [x15, #0x110]\n"
-      "fmla v11.4s, v7.4s, v0.s[2]\n"
+      "fmla v10.4s, v6.4s, v0.s[2]\n"
       "ldr q6, [x15, #0x120]\n"
-      "fmla v12.4s, v4.4s, v0.s[2]\n"
+      "fmla v11.4s, v7.4s, v0.s[2]\n"
       "ldr q7, [x15, #0x130]\n"
-      "fmla v13.4s, v5.4s, v0.s[2]\n"
+      "fmla v12.4s, v4.4s, v0.s[2]\n"
       "ldr q4, [x15, #0x140]\n"
-      "fmla v8.4s, v6.4s, v0.s[3]\n"
+      "fmla v13.4s, v5.4s, v0.s[2]\n"
       "ldr q5, [x15, #0x150]\n"
-      "fmla v9.4s, v7.4s, v0.s[3]\n"
+      "fmla v8.4s, v6.4s, v0.s[3]\n"
       "ldr q6, [x15, #0x160]\n"
-      "fmla v10.4s, v4.4s, v0.s[3]\n"
+      "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x15, #0x170]\n"
+      "add x11, x11, #0x10\n"
+      "sub x12, x12, #0x4\n"
+      "fmla v10.4s, v4.4s, v0.s[3]\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
-      "fmla v12.4s, v6.4s, v0.s[3]\n"
       "add x15, x15, #0x180\n"
+      "fmla v12.4s, v6.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
       "24:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x11, 26f\n"
+      "cbz x12, 26f\n"
       "25:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x10], #0x4\n"
-      "sub x11, x11, #0x1\n"
+      "ldr s0, [x11], #0x4\n"
+      "sub x12, x12, #0x1\n"
       "ldr q4, [x15, #0x0]\n"
-      "ldr q5, [x15, #0x10]\n"
-      "ldr q6, [x15, #0x20]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
+      "ldr q5, [x15, #0x10]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q4, [x15, #0x40]\n"
+      "ldr q6, [x15, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr q5, [x15, #0x50]\n"
+      "ldr q7, [x15, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x15, x15, #0x60\n"
+      "ldr q4, [x15, #0x40]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "ldr q5, [x15, #0x50]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
-      "cbnz x11, 25b\n"
+      "add x15, x15, #0x60\n"
+      "cbnz x12, 25b\n"
       "26:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x12, x12, #0x1\n"
-      "cmp x12, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x13, x13, #0x1\n"
+      "cmp x13, x20\n"
       "bne 19b\n"
-      "prfm pstl1keep, [x13, #0x0]\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
       "fmin v11.4s, v11.4s, v0.4s\n"
       "fmin v12.4s, v12.4s, v0.4s\n"
       "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
       "27:"  // Height 1: No activation
       "cmp x16, #0x18\n"
       "bge 40f\n"
       "tbz x16, #4, 31f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
-      "st1 { v11.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
+      "st1 { v11.4s }, [x14], #0x10\n"
       "tbz x16, #2, 29f\n"
-      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x14], #0x10\n"
       "tbz x16, #1, 28f\n"
-      "str d13, [x13], #0x8\n"
+      "str d13, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x14]\n"
       "b 39f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_20
       "tbz x16, #0, 39f\n"
-      "str s13, [x13, #0x0]\n"
+      "str s13, [x14, #0x0]\n"
       "b 39f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_16
       "tbz x16, #1, 30f\n"
-      "str d12, [x13], #0x8\n"
+      "str d12, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x14]\n"
       "b 39f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_16
       "tbz x16, #0, 39f\n"
-      "str s12, [x13, #0x0]\n"
+      "str s12, [x14, #0x0]\n"
       "b 39f\n"
       "31:"  // Height 1: Partial direct writeback: partial_8_0
       "tbz x16, #3, 35f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
       "tbz x16, #2, 33f\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
       "tbz x16, #1, 32f\n"
-      "str d11, [x13], #0x8\n"
+      "str d11, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x14]\n"
       "b 39f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x16, #0, 39f\n"
-      "str s11, [x13, #0x0]\n"
+      "str s11, [x14, #0x0]\n"
       "b 39f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x16, #1, 34f\n"
-      "str d10, [x13], #0x8\n"
+      "str d10, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x14]\n"
       "b 39f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x16, #0, 39f\n"
-      "str s10, [x13, #0x0]\n"
+      "str s10, [x14, #0x0]\n"
       "b 39f\n"
       "35:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x16, #2, 37f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
       "tbz x16, #1, 36f\n"
-      "str d9, [x13], #0x8\n"
+      "str d9, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x14]\n"
       "b 39f\n"
       "36:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x16, #0, 39f\n"
-      "str s9, [x13, #0x0]\n"
+      "str s9, [x14, #0x0]\n"
       "b 39f\n"
       "37:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x16, #1, 38f\n"
-      "str d8, [x13], #0x8\n"
+      "str d8, [x14], #0x8\n"
       "tbz x16, #0, 39f\n"
-      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x14]\n"
       "b 39f\n"
       "38:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x13, #0x0]\n"
+      "str s8, [x14, #0x0]\n"
       "39:"  // Height 1: Partial direct writeback: Done
       "b 41f\n"
       "40:"  // Height 1: Full writeback
-      "str q8, [x13, #0x0]\n"
-      "str q9, [x13, #0x10]\n"
-      "str q10, [x13, #0x20]\n"
-      "str q11, [x13, #0x30]\n"
-      "str q12, [x13, #0x40]\n"
-      "str q13, [x13, #0x50]\n"
-      "add x13, x13, #0x60\n"
+      "str q8, [x14, #0x0]\n"
+      "str q9, [x14, #0x10]\n"
+      "str q10, [x14, #0x20]\n"
+      "str q11, [x14, #0x30]\n"
+      "str q12, [x14, #0x40]\n"
+      "str q13, [x14, #0x50]\n"
+      "add x14, x14, #0x60\n"
       "41:"  // Height 1: Writeback done
       "subs x16, x16, #0x18\n"
       "bgt 2b\n"
       "b 166f\n"
       "42:"  // Height 2
+      "mov x17, %x[bias]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x14, %x[bias]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x13, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
       "43:"  // Height 2: Column loop
-      "cbz x14, 44f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
+      "cbz x17, 44f\n"
+      "ldr q8, [x17, #0x0]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q9, [x17, #0x10]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q12, [x14, #0x40]\n"
+      "ldr q10, [x17, #0x20]\n"
       "mov v16.16b, v10.16b\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q11, [x17, #0x30]\n"
       "mov v17.16b, v11.16b\n"
-      "add x14, x14, #0x60\n"
+      "ldr q12, [x17, #0x40]\n"
       "mov v18.16b, v12.16b\n"
+      "ldr q13, [x17, #0x50]\n"
       "mov v19.16b, v13.16b\n"
+      "add x17, x17, #0x60\n"
       "b 59f\n"
       "44:"  // Height 2: no bias
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x16, #0x18\n"
-      "add x23, x13, x19, LSL #2\n"
+      "add x23, x14, x20, LSL #2\n"
       "bge 57f\n"
       "tbz x16, #4, 48f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v11.4s }, [x14], #0x10\n"
       "ld1 { v17.4s }, [x23], #0x10\n"
       "tbz x16, #2, 46f\n"
-      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x14], #0x10\n"
       "ld1 { v18.4s }, [x23], #0x10\n"
       "tbz x16, #1, 45f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x13], #0x8\n"
+      "ldr d13, [x14], #0x8\n"
+      "mov x20, #0x58\n"
       "ldr d19, [x23], #0x8\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x14]\n"
       "ld1 { v19.s }[2], [x23]\n"
       "b 56f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x16, #0, 56f\n"
-      "ldr s13, [x13, #0x0]\n"
+      "ldr s13, [x14, #0x0]\n"
       "ldr s19, [x23, #0x0]\n"
       "b 56f\n"
       "46:"  // Height 2: Partial accumulate: partial_2_16
       "tbz x16, #1, 47f\n"
-      "ldr d12, [x13], #0x8\n"
+      "ldr d12, [x14], #0x8\n"
+      "mov x20, #0x48\n"
       "ldr d18, [x23], #0x8\n"
-      "mov x19, #0x48\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x14]\n"
       "ld1 { v18.s }[2], [x23]\n"
       "b 56f\n"
       "47:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x16, #0, 56f\n"
-      "ldr s12, [x13, #0x0]\n"
+      "ldr s12, [x14, #0x0]\n"
       "ldr s18, [x23, #0x0]\n"
       "b 56f\n"
       "48:"  // Height 2: Partial accumulate: partial_8_0
       "tbz x16, #3, 52f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
       "tbz x16, #2, 50f\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
       "tbz x16, #1, 49f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x13], #0x8\n"
+      "ldr d11, [x14], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d17, [x23], #0x8\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v11.s }[2], [x14]\n"
       "ld1 { v17.s }[2], [x23]\n"
       "b 56f\n"
       "49:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x16, #0, 56f\n"
-      "ldr s11, [x13, #0x0]\n"
+      "ldr s11, [x14, #0x0]\n"
       "ldr s17, [x23, #0x0]\n"
       "b 56f\n"
       "50:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x16, #1, 51f\n"
-      "ldr d10, [x13], #0x8\n"
+      "ldr d10, [x14], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d16, [x23], #0x8\n"
-      "mov x19, #0x28\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v10.s }[2], [x14]\n"
       "ld1 { v16.s }[2], [x23]\n"
       "b 56f\n"
       "51:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x16, #0, 56f\n"
-      "ldr s10, [x13, #0x0]\n"
+      "ldr s10, [x14, #0x0]\n"
       "ldr s16, [x23, #0x0]\n"
       "b 56f\n"
       "52:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x16, #2, 54f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "tbz x16, #1, 53f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x13], #0x8\n"
+      "ldr d9, [x14], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d15, [x23], #0x8\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v9.s }[2], [x14]\n"
       "ld1 { v15.s }[2], [x23]\n"
       "b 56f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x16, #0, 56f\n"
-      "ldr s9, [x13, #0x0]\n"
+      "ldr s9, [x14, #0x0]\n"
       "ldr s15, [x23, #0x0]\n"
       "b 56f\n"
       "54:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x16, #1, 55f\n"
-      "ldr d8, [x13], #0x8\n"
+      "ldr d8, [x14], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d14, [x23], #0x8\n"
-      "mov x19, #0x8\n"
       "tbz x16, #0, 56f\n"
-      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v8.s }[2], [x14]\n"
       "ld1 { v14.s }[2], [x23]\n"
       "b 56f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x13, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x14, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s14, [x23, #0x0]\n"
       "56:"  // Height 2: Partial accumulate: Done
-      "sub x13, x13, x19\n"
+      "sub x14, x14, x20\n"
       "b 59f\n"
       "57:"  // Height 2: full accumulate
-      "ldr q8, [x13, #0x0]\n"
-      "ldr q9, [x13, #0x10]\n"
-      "ldr q10, [x13, #0x20]\n"
-      "ldr q11, [x13, #0x30]\n"
-      "ldr q12, [x13, #0x40]\n"
-      "ldr q13, [x13, #0x50]\n"
+      "ldr q8, [x14, #0x0]\n"
+      "ldr q9, [x14, #0x10]\n"
+      "ldr q10, [x14, #0x20]\n"
+      "ldr q11, [x14, #0x30]\n"
+      "ldr q12, [x14, #0x40]\n"
+      "ldr q13, [x14, #0x50]\n"
       "ldr q14, [x23, #0x0]\n"
       "ldr q15, [x23, #0x10]\n"
       "ldr q16, [x23, #0x20]\n"
@@ -697,192 +697,192 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "59:"  // Height 2: setup done
-      "mov x12, #0x0\n"
+      "mov x13, #0x0\n"
       "60:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr w12, [x20, x13, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x10, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x12, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x10, x10, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x11, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x13, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 62f\n"
       "61:"  // Height 2: setup direct input
-      "mov x10, %x[input_ptr]\n"
-      "add x24, x10, x19, LSL #2\n"
+      "mov x11, %x[input_ptr]\n"
+      "add x25, x11, x20, LSL #2\n"
       "62:"  // Height 2: input setup done
-      "cmp x11, #0x4\n"
+      "cmp x12, #0x4\n"
       "blt 65f\n"
-      "ldr q0, [x10, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x11, #0x8\n"
+      "ldr q0, [x11, #0x0]\n"
+      "cmp x12, #0x8\n"
+      "ldr q1, [x25, #0x0]\n"
       "ldr q4, [x15, #0x0]\n"
+      "ldr q5, [x15, #0x10]\n"
+      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x30]\n"
       "blt 64f\n"
       "63:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr d5, [x15, #0x10]\n"
+      "ldr x10, [x15, #0x48]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr x9, [x15, #0x18]\n"
-      "ldr d6, [x15, #0x20]\n"
-      "add x10, x10, #0x10\n"
-      "ldr x28, [x15, #0x28]\n"
-      "add x24, x24, #0x10\n"
-      "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x30]\n"
-      "ldr x27, [x15, #0x38]\n"
-      "sub x11, x11, #0x4\n"
+      "ldr d4, [x15, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x9, [x15, #0x58]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr d4, [x15, #0x40]\n"
+      "ldr d5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x28, [x15, #0x68]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "ldr x26, [x15, #0x48]\n"
+      "ldr d6, [x15, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
+      "ldr x27, [x15, #0x78]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
-      "ldr d5, [x15, #0x50]\n"
-      "mov v4.d[1], x26\n"
-      "ldr x9, [x15, #0x58]\n"
-      "ldr d6, [x15, #0x60]\n"
-      "cmp x11, #0x8\n"
+      "ldr d7, [x15, #0x70]\n"
+      "mov v4.d[1], x10\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "ldr x28, [x15, #0x68]\n"
-      "fmla v18.4s, v4.4s, v1.s[0]\n"
       "mov v5.d[1], x9\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "fmla v18.4s, v4.4s, v1.s[0]\n"
+      "ldr d4, [x15, #0x80]\n"
       "mov v6.d[1], x28\n"
+      "mov v7.d[1], x27\n"
+      "ldr x10, [x15, #0x88]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "ldr x9, [x15, #0x98]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
-      "ldr d7, [x15, #0x70]\n"
+      "ldr d5, [x15, #0x90]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr x27, [x15, #0x78]\n"
+      "ldr x28, [x15, #0xa8]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr d4, [x15, #0x80]\n"
-      "ldr x26, [x15, #0x88]\n"
-      "mov v7.d[1], x27\n"
-      "ldr d5, [x15, #0x90]\n"
-      "ldr x9, [x15, #0x98]\n"
+      "ldr d6, [x15, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x27, [x15, #0xb8]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr d6, [x15, #0xa0]\n"
+      "ldr d7, [x15, #0xb0]\n"
+      "mov v4.d[1], x10\n"
       "fmla v10.4s, v4.4s, v0.s[1]\n"
       "mov v5.d[1], x9\n"
       "fmla v16.4s, v4.4s, v1.s[1]\n"
-      "ldr x28, [x15, #0xa8]\n"
+      "ldr d4, [x15, #0xc0]\n"
+      "mov v6.d[1], x28\n"
+      "mov v7.d[1], x27\n"
+      "ldr x10, [x15, #0xc8]\n"
       "fmla v11.4s, v5.4s, v0.s[1]\n"
-      "ldr d7, [x15, #0xb0]\n"
+      "ldr x9, [x15, #0xd8]\n"
       "fmla v17.4s, v5.4s, v1.s[1]\n"
-      "ldr x27, [x15, #0xb8]\n"
-      "mov v6.d[1], x28\n"
-      "ldr d4, [x15, #0xc0]\n"
-      "ldr x26, [x15, #0xc8]\n"
+      "ldr d5, [x15, #0xd0]\n"
       "fmla v12.4s, v6.4s, v0.s[1]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x28, [x15, #0xe8]\n"
       "fmla v18.4s, v6.4s, v1.s[1]\n"
-      "ldr d5, [x15, #0xd0]\n"
+      "ldr d6, [x15, #0xe0]\n"
       "fmla v13.4s, v7.4s, v0.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x27, [x15, #0xf8]\n"
       "fmla v19.4s, v7.4s, v1.s[1]\n"
-      "ldr x9, [x15, #0xd8]\n"
+      "ldr d7, [x15, #0xf0]\n"
+      "mov v4.d[1], x10\n"
       "fmla v8.4s, v4.4s, v0.s[2]\n"
-      "ldr d6, [x15, #0xe0]\n"
-      "fmla v14.4s, v4.4s, v1.s[2]\n"
-      "ldr x28, [x15, #0xe8]\n"
       "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0xf0]\n"
-      "ldr x27, [x15, #0xf8]\n"
-      "fmla v9.4s, v5.4s, v0.s[2]\n"
+      "fmla v14.4s, v4.4s, v1.s[2]\n"
+      "ldr d4, [x15, #0x100]\n"
       "mov v6.d[1], x28\n"
+      "mov v7.d[1], x27\n"
+      "ldr x10, [x15, #0x108]\n"
+      "fmla v9.4s, v5.4s, v0.s[2]\n"
+      "ldr x9, [x15, #0x118]\n"
       "fmla v15.4s, v5.4s, v1.s[2]\n"
-      "ldr d4, [x15, #0x100]\n"
+      "ldr d5, [x15, #0x110]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x28, [x15, #0x128]\n"
       "fmla v16.4s, v6.4s, v1.s[2]\n"
-      "ldr x26, [x15, #0x108]\n"
+      "ldr d6, [x15, #0x120]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr d5, [x15, #0x110]\n"
+      "ldr x27, [x15, #0x138]\n"
       "fmla v17.4s, v7.4s, v1.s[2]\n"
-      "ldr x9, [x15, #0x118]\n"
-      "mov v4.d[1], x26\n"
-      "ldr d6, [x15, #0x120]\n"
-      "ldr x28, [x15, #0x128]\n"
+      "ldr d7, [x15, #0x130]\n"
+      "mov v4.d[1], x10\n"
       "fmla v12.4s, v4.4s, v0.s[2]\n"
       "mov v5.d[1], x9\n"
       "fmla v18.4s, v4.4s, v1.s[2]\n"
-      "ldr d7, [x15, #0x130]\n"
-      "fmla v13.4s, v5.4s, v0.s[2]\n"
+      "ldr d4, [x15, #0x140]\n"
       "mov v6.d[1], x28\n"
+      "mov v7.d[1], x27\n"
+      "ldr x10, [x15, #0x148]\n"
+      "fmla v13.4s, v5.4s, v0.s[2]\n"
+      "ldr x9, [x15, #0x158]\n"
       "fmla v19.4s, v5.4s, v1.s[2]\n"
-      "ldr x27, [x15, #0x138]\n"
+      "ldr d5, [x15, #0x150]\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr d4, [x15, #0x140]\n"
+      "ldr x28, [x15, #0x168]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr x26, [x15, #0x148]\n"
-      "mov v7.d[1], x27\n"
-      "ldr d5, [x15, #0x150]\n"
-      "ldr x9, [x15, #0x158]\n"
+      "ldr d6, [x15, #0x160]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x27, [x15, #0x178]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr d6, [x15, #0x160]\n"
-      "fmla v10.4s, v4.4s, v0.s[3]\n"
+      "ldr d7, [x15, #0x170]\n"
+      "mov v4.d[1], x10\n"
+      "add x11, x11, #0x10\n"
       "mov v5.d[1], x9\n"
+      "add x25, x25, #0x10\n"
+      "mov v6.d[1], x28\n"
+      "add x15, x15, #0x180\n"
+      "mov v7.d[1], x27\n"
+      "fmla v10.4s, v4.4s, v0.s[3]\n"
       "fmla v16.4s, v4.4s, v1.s[3]\n"
-      "ldr x28, [x15, #0x168]\n"
+      "ldr d4, [x15, #0x0]\n"
+      "ldr x10, [x15, #0x8]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
-      "ldr d7, [x15, #0x170]\n"
       "fmla v17.4s, v5.4s, v1.s[3]\n"
-      "ldr x27, [x15, #0x178]\n"
-      "mov v6.d[1], x28\n"
-      "ldr x25, [x10, #0x8]\n"
-      "ldr x23, [x24, #0x8]\n"
-      "add x15, x15, #0x180\n"
+      "ldr d5, [x15, #0x10]\n"
+      "ldr x9, [x15, #0x18]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
-      "mov v7.d[1], x27\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
-      "ldr d4, [x15, #0x0]\n"
+      "ldr d6, [x15, #0x20]\n"
+      "ldr x28, [x15, #0x28]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr x26, [x15, #0x8]\n"
+      "ldr d0, [x11, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr d0, [x10, #0x0]\n"
-      "ldr d1, [x24, #0x0]\n"
-      "mov v4.d[1], x26\n"
-      "mov v0.d[1], x25\n"
-      "mov v1.d[1], x23\n"
+      "ldr d1, [x25, #0x0]\n"
+      "sub x12, x12, #0x4\n"
+      "ldr d7, [x15, #0x30]\n"
+      "cmp x12, #0x8\n"
+      "ldr x26, [x11, #0x8]\n"
+      "mov v4.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
+      "mov v5.d[1], x9\n"
+      "ldr x27, [x15, #0x38]\n"
+      "mov v6.d[1], x28\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
+      "mov v0.d[1], x26\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "mov v1.d[1], x24\n"
+      "mov v7.d[1], x27\n"
       "bge 63b\n"
       "64:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x15, #0x10]\n"
+      "add x11, x11, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x15, #0x20]\n"
-      "ldr q7, [x15, #0x30]\n"
-      "sub x11, x11, #0x4\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x15, #0x40]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
       "ldr q5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x10, x10, #0x10\n"
+      "sub x12, x12, #0x4\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x15, #0x60]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x15, #0x70]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x15, #0x80]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "ldr q5, [x15, #0x90]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
@@ -937,45 +937,43 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmla v13.4s, v7.4s, v0.s[3]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
       "65:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x11, 67f\n"
+      "cbz x12, 67f\n"
       "66:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x10], #0x4\n"
-      "sub x11, x11, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
+      "ldr s0, [x11], #0x4\n"
+      "sub x12, x12, #0x1\n"
+      "ldr s1, [x25], #0x4\n"
       "ldr q4, [x15, #0x0]\n"
-      "ldr q5, [x15, #0x10]\n"
-      "ldr q6, [x15, #0x20]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
+      "ldr q5, [x15, #0x10]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q4, [x15, #0x40]\n"
+      "ldr q6, [x15, #0x20]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "ldr q7, [x15, #0x30]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr q5, [x15, #0x50]\n"
+      "ldr q4, [x15, #0x40]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x15, x15, #0x60\n"
+      "ldr q5, [x15, #0x50]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "add x15, x15, #0x60\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
-      "cbnz x11, 66b\n"
+      "cbnz x12, 66b\n"
       "67:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x12, x12, #0x1\n"
-      "cmp x12, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x13, x13, #0x1\n"
+      "cmp x13, x20\n"
       "bne 60b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x13, #0x0]\n"
-      "add x23, x13, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x14, x20, LSL #2\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 68f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -986,131 +984,133 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "68:"  // Height 2: No activation
       "cmp x16, #0x18\n"
       "bge 81f\n"
       "tbz x16, #4, 72f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
-      "st1 { v11.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
+      "st1 { v11.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
       "st1 { v17.4s }, [x23], #0x10\n"
       "tbz x16, #2, 70f\n"
-      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x14], #0x10\n"
       "st1 { v18.4s }, [x23], #0x10\n"
       "tbz x16, #1, 69f\n"
-      "str d13, [x13], #0x8\n"
+      "str d13, [x14], #0x8\n"
       "str d19, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x14]\n"
       "st1 { v19.s }[2], [x23]\n"
       "b 80f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_20
       "tbz x16, #0, 80f\n"
-      "str s13, [x13, #0x0]\n"
+      "str s13, [x14, #0x0]\n"
       "str s19, [x23, #0x0]\n"
       "b 80f\n"
       "70:"  // Height 2: Partial direct writeback: partial_2_16
       "tbz x16, #1, 71f\n"
-      "str d12, [x13], #0x8\n"
+      "str d12, [x14], #0x8\n"
       "str d18, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x14]\n"
       "st1 { v18.s }[2], [x23]\n"
       "b 80f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_16
       "tbz x16, #0, 80f\n"
-      "str s12, [x13, #0x0]\n"
+      "str s12, [x14, #0x0]\n"
       "str s18, [x23, #0x0]\n"
       "b 80f\n"
       "72:"  // Height 2: Partial direct writeback: partial_8_0
       "tbz x16, #3, 76f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "tbz x16, #2, 74f\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
       "tbz x16, #1, 73f\n"
-      "str d11, [x13], #0x8\n"
+      "str d11, [x14], #0x8\n"
       "str d17, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x14]\n"
       "st1 { v17.s }[2], [x23]\n"
       "b 80f\n"
       "73:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x16, #0, 80f\n"
-      "str s11, [x13, #0x0]\n"
+      "str s11, [x14, #0x0]\n"
       "str s17, [x23, #0x0]\n"
       "b 80f\n"
       "74:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x16, #1, 75f\n"
-      "str d10, [x13], #0x8\n"
+      "str d10, [x14], #0x8\n"
       "str d16, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x14]\n"
       "st1 { v16.s }[2], [x23]\n"
       "b 80f\n"
       "75:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x16, #0, 80f\n"
-      "str s10, [x13, #0x0]\n"
+      "str s10, [x14, #0x0]\n"
       "str s16, [x23, #0x0]\n"
       "b 80f\n"
       "76:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x16, #2, 78f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "tbz x16, #1, 77f\n"
-      "str d9, [x13], #0x8\n"
+      "str d9, [x14], #0x8\n"
       "str d15, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x14]\n"
       "st1 { v15.s }[2], [x23]\n"
       "b 80f\n"
       "77:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x16, #0, 80f\n"
-      "str s9, [x13, #0x0]\n"
+      "str s9, [x14, #0x0]\n"
       "str s15, [x23, #0x0]\n"
       "b 80f\n"
       "78:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x16, #1, 79f\n"
-      "str d8, [x13], #0x8\n"
+      "str d8, [x14], #0x8\n"
       "str d14, [x23], #0x8\n"
       "tbz x16, #0, 80f\n"
-      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x14]\n"
       "st1 { v14.s }[2], [x23]\n"
       "b 80f\n"
       "79:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x13, #0x0]\n"
+      "str s8, [x14, #0x0]\n"
       "str s14, [x23, #0x0]\n"
       "80:"  // Height 2: Partial direct writeback: Done
       "b 82f\n"
       "81:"  // Height 2: Full writeback
-      "str q8, [x13, #0x0]\n"
-      "str q9, [x13, #0x10]\n"
-      "str q10, [x13, #0x20]\n"
-      "str q11, [x13, #0x30]\n"
-      "str q12, [x13, #0x40]\n"
-      "str q13, [x13, #0x50]\n"
-      "add x13, x13, #0x60\n"
+      "str q8, [x14, #0x0]\n"
+      "str q9, [x14, #0x10]\n"
+      "str q10, [x14, #0x20]\n"
+      "str q11, [x14, #0x30]\n"
+      "str q12, [x14, #0x40]\n"
+      "str q13, [x14, #0x50]\n"
+      "add x14, x14, #0x60\n"
       "str q14, [x23, #0x0]\n"
       "str q15, [x23, #0x10]\n"
       "str q16, [x23, #0x20]\n"
@@ -1122,26 +1122,26 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "bgt 43b\n"
       "b 166f\n"
       "83:"  // Height 3
+      "mov x17, %x[bias]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x14, %x[bias]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x13, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
       "84:"  // Height 3: Column loop
-      "cbz x14, 85f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
+      "cbz x17, 85f\n"
+      "ldr q8, [x17, #0x0]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q9, [x17, #0x10]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q12, [x14, #0x40]\n"
+      "ldr q10, [x17, #0x20]\n"
       "mov v16.16b, v10.16b\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q11, [x17, #0x30]\n"
       "mov v17.16b, v11.16b\n"
-      "add x14, x14, #0x60\n"
+      "ldr q12, [x17, #0x40]\n"
       "mov v18.16b, v12.16b\n"
+      "ldr q13, [x17, #0x50]\n"
       "mov v19.16b, v13.16b\n"
       "mov v20.16b, v8.16b\n"
+      "add x17, x17, #0x60\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
@@ -1150,158 +1150,158 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "b 100f\n"
       "85:"  // Height 3: no bias
       "tbz %x[flags], #0, 99f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x14, x20, LSL #2\n"
       "cmp x16, #0x18\n"
-      "add x23, x13, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 98f\n"
       "tbz x16, #4, 89f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
       "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
       "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v11.4s }, [x14], #0x10\n"
       "ld1 { v17.4s }, [x23], #0x10\n"
       "ld1 { v23.4s }, [x22], #0x10\n"
       "tbz x16, #2, 87f\n"
-      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x14], #0x10\n"
       "ld1 { v18.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x16, #1, 86f\n"
-      "ldr d13, [x13], #0x8\n"
-      "mov x19, #0x58\n"
+      "ldr d13, [x14], #0x8\n"
+      "mov x20, #0x58\n"
       "ldr d19, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x14]\n"
       "ld1 { v19.s }[2], [x23]\n"
       "ld1 { v25.s }[2], [x22]\n"
       "b 97f\n"
       "86:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x16, #0, 97f\n"
-      "ldr s13, [x13, #0x0]\n"
+      "ldr s13, [x14, #0x0]\n"
       "ldr s19, [x23, #0x0]\n"
       "ldr s25, [x22, #0x0]\n"
       "b 97f\n"
       "87:"  // Height 3: Partial accumulate: partial_2_16
       "tbz x16, #1, 88f\n"
-      "ldr d12, [x13], #0x8\n"
+      "ldr d12, [x14], #0x8\n"
+      "mov x20, #0x48\n"
       "ldr d18, [x23], #0x8\n"
-      "mov x19, #0x48\n"
       "ldr d24, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x14]\n"
       "ld1 { v18.s }[2], [x23]\n"
       "ld1 { v24.s }[2], [x22]\n"
       "b 97f\n"
       "88:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x16, #0, 97f\n"
-      "ldr s12, [x13, #0x0]\n"
+      "ldr s12, [x14, #0x0]\n"
       "ldr s18, [x23, #0x0]\n"
       "ldr s24, [x22, #0x0]\n"
       "b 97f\n"
       "89:"  // Height 3: Partial accumulate: partial_8_0
       "tbz x16, #3, 93f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
       "ld1 { v21.4s }, [x22], #0x10\n"
       "tbz x16, #2, 91f\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
       "ld1 { v22.4s }, [x22], #0x10\n"
       "tbz x16, #1, 90f\n"
-      "ldr d11, [x13], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x14], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d17, [x23], #0x8\n"
       "ldr d23, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v11.s }[2], [x14]\n"
       "ld1 { v17.s }[2], [x23]\n"
       "ld1 { v23.s }[2], [x22]\n"
       "b 97f\n"
       "90:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x16, #0, 97f\n"
-      "ldr s11, [x13, #0x0]\n"
+      "ldr s11, [x14, #0x0]\n"
       "ldr s17, [x23, #0x0]\n"
       "ldr s23, [x22, #0x0]\n"
       "b 97f\n"
       "91:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x16, #1, 92f\n"
-      "ldr d10, [x13], #0x8\n"
+      "ldr d10, [x14], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d16, [x23], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d22, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v10.s }[2], [x14]\n"
       "ld1 { v16.s }[2], [x23]\n"
       "ld1 { v22.s }[2], [x22]\n"
       "b 97f\n"
       "92:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x16, #0, 97f\n"
-      "ldr s10, [x13, #0x0]\n"
+      "ldr s10, [x14, #0x0]\n"
       "ldr s16, [x23, #0x0]\n"
       "ldr s22, [x22, #0x0]\n"
       "b 97f\n"
       "93:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x16, #2, 95f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
       "tbz x16, #1, 94f\n"
-      "ldr d9, [x13], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x14], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d15, [x23], #0x8\n"
       "ldr d21, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v9.s }[2], [x14]\n"
       "ld1 { v15.s }[2], [x23]\n"
       "ld1 { v21.s }[2], [x22]\n"
       "b 97f\n"
       "94:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x16, #0, 97f\n"
-      "ldr s9, [x13, #0x0]\n"
+      "ldr s9, [x14, #0x0]\n"
       "ldr s15, [x23, #0x0]\n"
       "ldr s21, [x22, #0x0]\n"
       "b 97f\n"
       "95:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x16, #1, 96f\n"
-      "ldr d8, [x13], #0x8\n"
+      "ldr d8, [x14], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d14, [x23], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d20, [x22], #0x8\n"
       "tbz x16, #0, 97f\n"
-      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v8.s }[2], [x14]\n"
       "ld1 { v14.s }[2], [x23]\n"
       "ld1 { v20.s }[2], [x22]\n"
       "b 97f\n"
       "96:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x13, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x14, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s14, [x23, #0x0]\n"
       "ldr s20, [x22, #0x0]\n"
       "97:"  // Height 3: Partial accumulate: Done
-      "sub x13, x13, x19\n"
+      "sub x14, x14, x20\n"
       "b 100f\n"
       "98:"  // Height 3: full accumulate
-      "ldr q8, [x13, #0x0]\n"
-      "ldr q9, [x13, #0x10]\n"
-      "ldr q10, [x13, #0x20]\n"
-      "ldr q11, [x13, #0x30]\n"
-      "ldr q12, [x13, #0x40]\n"
-      "ldr q13, [x13, #0x50]\n"
+      "ldr q8, [x14, #0x0]\n"
+      "ldr q9, [x14, #0x10]\n"
+      "ldr q10, [x14, #0x20]\n"
+      "ldr q11, [x14, #0x30]\n"
+      "ldr q12, [x14, #0x40]\n"
+      "ldr q13, [x14, #0x50]\n"
       "ldr q14, [x23, #0x0]\n"
       "ldr q15, [x23, #0x10]\n"
       "ldr q16, [x23, #0x20]\n"
@@ -1335,229 +1335,229 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "movi v24.16b, #0x0\n"
       "movi v25.16b, #0x0\n"
       "100:"  // Height 3: setup done
-      "mov x12, #0x0\n"
+      "mov x13, #0x0\n"
       "101:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr w12, [x20, x13, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 102f\n"
-      "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x10, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x22, [x20, #0x10]\n"
-      "cbnz x12, 103f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x10, x10, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x11, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x23, [x21, #0x10]\n"
+      "cbnz x13, 103f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 103f\n"
       "102:"  // Height 3: setup direct input
-      "mov x10, %x[input_ptr]\n"
-      "add x24, x10, x19, LSL #2\n"
-      "add x22, x24, x19, LSL #2\n"
+      "mov x11, %x[input_ptr]\n"
+      "add x25, x11, x20, LSL #2\n"
+      "add x23, x25, x20, LSL #2\n"
       "103:"  // Height 3: input setup done
-      "cmp x11, #0x4\n"
+      "cmp x12, #0x4\n"
       "blt 106f\n"
-      "ldr q0, [x10, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x11, #0x8\n"
-      "ldr q2, [x22, #0x0]\n"
+      "ldr q0, [x11, #0x0]\n"
+      "cmp x12, #0x8\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
       "ldr q4, [x15, #0x0]\n"
+      "ldr q5, [x15, #0x10]\n"
+      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x30]\n"
       "blt 105f\n"
       "104:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr d5, [x15, #0x10]\n"
+      "ldr x10, [x15, #0x48]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr x9, [x15, #0x18]\n"
+      "ldr x9, [x15, #0x58]\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr d6, [x15, #0x20]\n"
-      "ldr x28, [x15, #0x28]\n"
-      "add x10, x10, #0x10\n"
-      "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x30]\n"
-      "ldr x27, [x15, #0x38]\n"
-      "add x24, x24, #0x10\n"
+      "ldr d4, [x15, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x28, [x15, #0x68]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr d4, [x15, #0x40]\n"
+      "ldr x27, [x15, #0x78]\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "mov v7.d[1], x27\n"
+      "ldr d5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr x26, [x15, #0x48]\n"
+      "mov v4.d[1], x10\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
+      "mov v5.d[1], x9\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
-      "ldr d5, [x15, #0x50]\n"
+      "ldr d6, [x15, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "mov v4.d[1], x26\n"
+      "mov v6.d[1], x28\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
-      "ldr x9, [x15, #0x58]\n"
+      "ldr x10, [x15, #0x88]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr d7, [x15, #0x70]\n"
+      "mov v7.d[1], x27\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "ldr d6, [x15, #0x60]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
-      "mov v5.d[1], x9\n"
+      "ldr x9, [x15, #0x98]\n"
       "fmla v24.4s, v4.4s, v2.s[0]\n"
-      "ldr x28, [x15, #0x68]\n"
+      "ldr d4, [x15, #0x80]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
-      "ldr d7, [x15, #0x70]\n"
+      "ldr x28, [x15, #0xa8]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
-      "ldr x27, [x15, #0x78]\n"
+      "ldr x27, [x15, #0xb8]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
-      "mov v6.d[1], x28\n"
-      "ldr d4, [x15, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "ldr d5, [x15, #0x90]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "mov v5.d[1], x9\n"
       "fmla v20.4s, v6.4s, v2.s[1]\n"
-      "ldr x26, [x15, #0x88]\n"
+      "ldr d6, [x15, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr d5, [x15, #0x90]\n"
+      "mov v6.d[1], x28\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr x9, [x15, #0x98]\n"
+      "ldr x10, [x15, #0xc8]\n"
       "fmla v21.4s, v7.4s, v2.s[1]\n"
-      "mov v4.d[1], x26\n"
-      "ldr d6, [x15, #0xa0]\n"
-      "sub x11, x11, #0x4\n"
+      "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x27\n"
       "fmla v10.4s, v4.4s, v0.s[1]\n"
-      "mov v5.d[1], x9\n"
       "fmla v16.4s, v4.4s, v1.s[1]\n"
-      "ldr x28, [x15, #0xa8]\n"
+      "ldr x9, [x15, #0xd8]\n"
       "fmla v22.4s, v4.4s, v2.s[1]\n"
-      "ldr d7, [x15, #0xb0]\n"
+      "ldr d4, [x15, #0xc0]\n"
       "fmla v11.4s, v5.4s, v0.s[1]\n"
-      "ldr x27, [x15, #0xb8]\n"
+      "ldr x28, [x15, #0xe8]\n"
       "fmla v17.4s, v5.4s, v1.s[1]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0xf8]\n"
       "fmla v23.4s, v5.4s, v2.s[1]\n"
-      "ldr d4, [x15, #0xc0]\n"
+      "ldr d5, [x15, #0xd0]\n"
       "fmla v12.4s, v6.4s, v0.s[1]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v18.4s, v6.4s, v1.s[1]\n"
-      "ldr x26, [x15, #0xc8]\n"
+      "mov v5.d[1], x9\n"
       "fmla v24.4s, v6.4s, v2.s[1]\n"
-      "ldr d5, [x15, #0xd0]\n"
+      "ldr d6, [x15, #0xe0]\n"
       "fmla v13.4s, v7.4s, v0.s[1]\n"
-      "ldr x9, [x15, #0xd8]\n"
+      "mov v6.d[1], x28\n"
       "fmla v19.4s, v7.4s, v1.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x10, [x15, #0x108]\n"
       "fmla v25.4s, v7.4s, v2.s[1]\n"
-      "ldr d6, [x15, #0xe0]\n"
+      "ldr d7, [x15, #0xf0]\n"
+      "mov v7.d[1], x27\n"
       "fmla v8.4s, v4.4s, v0.s[2]\n"
-      "mov v5.d[1], x9\n"
       "fmla v14.4s, v4.4s, v1.s[2]\n"
-      "ldr x28, [x15, #0xe8]\n"
+      "ldr x9, [x15, #0x118]\n"
       "fmla v20.4s, v4.4s, v2.s[2]\n"
-      "ldr d7, [x15, #0xf0]\n"
+      "ldr d4, [x15, #0x100]\n"
       "fmla v9.4s, v5.4s, v0.s[2]\n"
-      "ldr x27, [x15, #0xf8]\n"
+      "ldr x28, [x15, #0x128]\n"
       "fmla v15.4s, v5.4s, v1.s[2]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0x138]\n"
       "fmla v21.4s, v5.4s, v2.s[2]\n"
-      "ldr d4, [x15, #0x100]\n"
+      "ldr d5, [x15, #0x110]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v16.4s, v6.4s, v1.s[2]\n"
-      "ldr x26, [x15, #0x108]\n"
+      "mov v5.d[1], x9\n"
       "fmla v22.4s, v6.4s, v2.s[2]\n"
-      "ldr d5, [x15, #0x110]\n"
+      "ldr d6, [x15, #0x120]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr x9, [x15, #0x118]\n"
+      "mov v6.d[1], x28\n"
       "fmla v17.4s, v7.4s, v1.s[2]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x10, [x15, #0x148]\n"
       "fmla v23.4s, v7.4s, v2.s[2]\n"
-      "ldr d6, [x15, #0x120]\n"
+      "ldr d7, [x15, #0x130]\n"
+      "mov v7.d[1], x27\n"
       "fmla v12.4s, v4.4s, v0.s[2]\n"
-      "mov v5.d[1], x9\n"
       "fmla v18.4s, v4.4s, v1.s[2]\n"
-      "ldr x28, [x15, #0x128]\n"
+      "ldr x9, [x15, #0x158]\n"
       "fmla v24.4s, v4.4s, v2.s[2]\n"
-      "ldr d7, [x15, #0x130]\n"
+      "ldr d4, [x15, #0x140]\n"
       "fmla v13.4s, v5.4s, v0.s[2]\n"
-      "ldr x27, [x15, #0x138]\n"
+      "ldr x28, [x15, #0x168]\n"
       "fmla v19.4s, v5.4s, v1.s[2]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0x178]\n"
       "fmla v25.4s, v5.4s, v2.s[2]\n"
-      "ldr d4, [x15, #0x140]\n"
+      "ldr d5, [x15, #0x150]\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr x26, [x15, #0x148]\n"
+      "mov v5.d[1], x9\n"
       "fmla v20.4s, v6.4s, v2.s[3]\n"
-      "ldr d5, [x15, #0x150]\n"
+      "ldr d6, [x15, #0x160]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "ldr x9, [x15, #0x158]\n"
+      "mov v6.d[1], x28\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "mov v4.d[1], x26\n"
+      "add x11, x11, #0x10\n"
       "fmla v21.4s, v7.4s, v2.s[3]\n"
-      "ldr d6, [x15, #0x160]\n"
+      "ldr d7, [x15, #0x170]\n"
+      "mov v7.d[1], x27\n"
+      "add x25, x25, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "add x15, x15, #0x180\n"
       "fmla v10.4s, v4.4s, v0.s[3]\n"
-      "mov v5.d[1], x9\n"
+      "ldr x10, [x15, #0x8]\n"
       "fmla v16.4s, v4.4s, v1.s[3]\n"
-      "ldr x28, [x15, #0x168]\n"
+      "ldr x9, [x15, #0x18]\n"
       "fmla v22.4s, v4.4s, v2.s[3]\n"
-      "ldr d7, [x15, #0x170]\n"
+      "ldr d4, [x15, #0x0]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
-      "ldr x27, [x15, #0x178]\n"
+      "ldr x28, [x15, #0x28]\n"
       "fmla v17.4s, v5.4s, v1.s[3]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x26, [x11, #0x8]\n"
       "fmla v23.4s, v5.4s, v2.s[3]\n"
-      "ldr x25, [x10, #0x8]\n"
+      "ldr d5, [x15, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v24.4s, v6.4s, v2.s[3]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr d6, [x15, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x10, #0x0]\n"
+      "ldr d0, [x11, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x24, #0x0]\n"
+      "ldr d1, [x25, #0x0]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
-      "ldr d2, [x22, #0x0]\n"
-      "mov v0.d[1], x25\n"
-      "cmp x11, #0x8\n"
-      "mov v1.d[1], x23\n"
-      "add x15, x15, #0x180\n"
-      "mov v2.d[1], x21\n"
-      "ldr d4, [x15, #0x0]\n"
-      "ldr x26, [x15, #0x8]\n"
-      "mov v4.d[1], x26\n"
+      "ldr d2, [x23, #0x0]\n"
+      "ldr d7, [x15, #0x30]\n"
+      "sub x12, x12, #0x4\n"
+      "ldr x27, [x15, #0x38]\n"
+      "cmp x12, #0x8\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
+      "mov v4.d[1], x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "mov v5.d[1], x9\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "mov v6.d[1], x28\n"
+      "mov v0.d[1], x26\n"
+      "mov v1.d[1], x24\n"
+      "mov v2.d[1], x22\n"
+      "mov v7.d[1], x27\n"
       "bge 104b\n"
       "105:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x15, #0x10]\n"
+      "add x11, x11, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x15, #0x20]\n"
+      "add x25, x25, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x15, #0x40]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "sub x11, x11, #0x4\n"
+      "sub x12, x12, #0x4\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
       "ldr q5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x10, x10, #0x10\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x15, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x15, #0x70]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "add x22, x22, #0x10\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v24.4s, v4.4s, v2.s[0]\n"
       "ldr q4, [x15, #0x80]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
@@ -1634,21 +1634,21 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmla v19.4s, v7.4s, v1.s[3]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
       "106:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x11, 108f\n"
+      "cbz x12, 108f\n"
       "107:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x10], #0x4\n"
-      "sub x11, x11, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
+      "ldr s0, [x11], #0x4\n"
+      "sub x12, x12, #0x1\n"
+      "ldr s1, [x25], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
       "ldr q4, [x15, #0x0]\n"
-      "ldr q5, [x15, #0x10]\n"
-      "ldr q6, [x15, #0x20]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
+      "ldr q5, [x15, #0x10]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
+      "ldr q6, [x15, #0x20]\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q4, [x15, #0x40]\n"
+      "ldr q7, [x15, #0x30]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "ldr q4, [x15, #0x40]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
       "ldr q5, [x15, #0x50]\n"
@@ -1665,23 +1665,21 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
-      "cbnz x11, 107b\n"
+      "cbnz x12, 107b\n"
       "108:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x12, x12, #0x1\n"
-      "cmp x12, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x13, x13, #0x1\n"
+      "cmp x13, x20\n"
       "bne 101b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x13, #0x0]\n"
-      "add x23, x13, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x14, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 109f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -1692,16 +1690,6 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
       "fmin v20.4s, v20.4s, v0.4s\n"
@@ -1710,22 +1698,34 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmin v23.4s, v23.4s, v0.4s\n"
       "fmin v24.4s, v24.4s, v0.4s\n"
       "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
       "109:"  // Height 3: No activation
       "cmp x16, #0x18\n"
       "bge 122f\n"
       "tbz x16, #4, 113f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
-      "st1 { v11.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
+      "st1 { v11.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
@@ -1735,127 +1735,127 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "st1 { v22.4s }, [x22], #0x10\n"
       "st1 { v23.4s }, [x22], #0x10\n"
       "tbz x16, #2, 111f\n"
-      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x14], #0x10\n"
       "st1 { v18.4s }, [x23], #0x10\n"
       "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x16, #1, 110f\n"
-      "str d13, [x13], #0x8\n"
+      "str d13, [x14], #0x8\n"
       "str d19, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x14]\n"
       "st1 { v19.s }[2], [x23]\n"
       "st1 { v25.s }[2], [x22]\n"
       "b 121f\n"
       "110:"  // Height 3: Partial direct writeback: partial_1_20
       "tbz x16, #0, 121f\n"
-      "str s13, [x13, #0x0]\n"
+      "str s13, [x14, #0x0]\n"
       "str s19, [x23, #0x0]\n"
       "str s25, [x22, #0x0]\n"
       "b 121f\n"
       "111:"  // Height 3: Partial direct writeback: partial_2_16
       "tbz x16, #1, 112f\n"
-      "str d12, [x13], #0x8\n"
+      "str d12, [x14], #0x8\n"
       "str d18, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x14]\n"
       "st1 { v18.s }[2], [x23]\n"
       "st1 { v24.s }[2], [x22]\n"
       "b 121f\n"
       "112:"  // Height 3: Partial direct writeback: partial_1_16
       "tbz x16, #0, 121f\n"
-      "str s12, [x13, #0x0]\n"
+      "str s12, [x14, #0x0]\n"
       "str s18, [x23, #0x0]\n"
       "str s24, [x22, #0x0]\n"
       "b 121f\n"
       "113:"  // Height 3: Partial direct writeback: partial_8_0
       "tbz x16, #3, 117f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "st1 { v20.4s }, [x22], #0x10\n"
       "st1 { v21.4s }, [x22], #0x10\n"
       "tbz x16, #2, 115f\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
       "st1 { v22.4s }, [x22], #0x10\n"
       "tbz x16, #1, 114f\n"
-      "str d11, [x13], #0x8\n"
+      "str d11, [x14], #0x8\n"
       "str d17, [x23], #0x8\n"
       "str d23, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x14]\n"
       "st1 { v17.s }[2], [x23]\n"
       "st1 { v23.s }[2], [x22]\n"
       "b 121f\n"
       "114:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x16, #0, 121f\n"
-      "str s11, [x13, #0x0]\n"
+      "str s11, [x14, #0x0]\n"
       "str s17, [x23, #0x0]\n"
       "str s23, [x22, #0x0]\n"
       "b 121f\n"
       "115:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x16, #1, 116f\n"
-      "str d10, [x13], #0x8\n"
+      "str d10, [x14], #0x8\n"
       "str d16, [x23], #0x8\n"
       "str d22, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x14]\n"
       "st1 { v16.s }[2], [x23]\n"
       "st1 { v22.s }[2], [x22]\n"
       "b 121f\n"
       "116:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x16, #0, 121f\n"
-      "str s10, [x13, #0x0]\n"
+      "str s10, [x14, #0x0]\n"
       "str s16, [x23, #0x0]\n"
       "str s22, [x22, #0x0]\n"
       "b 121f\n"
       "117:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x16, #2, 119f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v20.4s }, [x22], #0x10\n"
       "tbz x16, #1, 118f\n"
-      "str d9, [x13], #0x8\n"
+      "str d9, [x14], #0x8\n"
       "str d15, [x23], #0x8\n"
       "str d21, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x14]\n"
       "st1 { v15.s }[2], [x23]\n"
       "st1 { v21.s }[2], [x22]\n"
       "b 121f\n"
       "118:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x16, #0, 121f\n"
-      "str s9, [x13, #0x0]\n"
+      "str s9, [x14, #0x0]\n"
       "str s15, [x23, #0x0]\n"
       "str s21, [x22, #0x0]\n"
       "b 121f\n"
       "119:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x16, #1, 120f\n"
-      "str d8, [x13], #0x8\n"
+      "str d8, [x14], #0x8\n"
       "str d14, [x23], #0x8\n"
       "str d20, [x22], #0x8\n"
       "tbz x16, #0, 121f\n"
-      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x14]\n"
       "st1 { v14.s }[2], [x23]\n"
       "st1 { v20.s }[2], [x22]\n"
       "b 121f\n"
       "120:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x13, #0x0]\n"
+      "str s8, [x14, #0x0]\n"
       "str s14, [x23, #0x0]\n"
       "str s20, [x22, #0x0]\n"
       "121:"  // Height 3: Partial direct writeback: Done
       "b 123f\n"
       "122:"  // Height 3: Full writeback
-      "str q8, [x13, #0x0]\n"
-      "str q9, [x13, #0x10]\n"
-      "str q10, [x13, #0x20]\n"
-      "str q11, [x13, #0x30]\n"
-      "str q12, [x13, #0x40]\n"
-      "str q13, [x13, #0x50]\n"
-      "add x13, x13, #0x60\n"
+      "str q8, [x14, #0x0]\n"
+      "str q9, [x14, #0x10]\n"
+      "str q10, [x14, #0x20]\n"
+      "str q11, [x14, #0x30]\n"
+      "str q12, [x14, #0x40]\n"
+      "str q13, [x14, #0x50]\n"
+      "add x14, x14, #0x60\n"
       "str q14, [x23, #0x0]\n"
       "str q15, [x23, #0x10]\n"
       "str q16, [x23, #0x20]\n"
@@ -1873,29 +1873,29 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "bgt 84b\n"
       "b 166f\n"
       "124:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x10\n"
+      "mov x17, %x[bias]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x14, %x[bias]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x13, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x10\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "125:"  // Height 4: Column loop
-      "cbz x14, 126f\n"
-      "ldr q8, [x14, #0x0]\n"
-      "ldr q9, [x14, #0x10]\n"
-      "ldr q10, [x14, #0x20]\n"
+      "cbz x17, 126f\n"
+      "ldr q8, [x17, #0x0]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q11, [x14, #0x30]\n"
+      "ldr q9, [x17, #0x10]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q12, [x14, #0x40]\n"
+      "ldr q10, [x17, #0x20]\n"
       "mov v16.16b, v10.16b\n"
-      "ldr q13, [x14, #0x50]\n"
+      "ldr q11, [x17, #0x30]\n"
       "mov v17.16b, v11.16b\n"
-      "add x14, x14, #0x60\n"
+      "ldr q12, [x17, #0x40]\n"
       "mov v18.16b, v12.16b\n"
+      "ldr q13, [x17, #0x50]\n"
       "mov v19.16b, v13.16b\n"
       "mov v20.16b, v8.16b\n"
+      "add x17, x17, #0x60\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
@@ -1910,186 +1910,186 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "b 141f\n"
       "126:"  // Height 4: no bias
       "tbz %x[flags], #0, 140f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x14, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x16, #0x18\n"
-      "add x23, x13, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 139f\n"
       "tbz x16, #4, 130f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
       "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
       "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v11.4s }, [x13], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x14], #0x10\n"
       "ld1 { v17.4s }, [x23], #0x10\n"
       "ld1 { v23.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x21], #0x10\n"
       "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x16, #2, 128f\n"
-      "ld1 { v12.4s }, [x13], #0x10\n"
+      "ld1 { v12.4s }, [x14], #0x10\n"
       "ld1 { v18.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
       "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x16, #1, 127f\n"
-      "ldr d13, [x13], #0x8\n"
-      "mov x19, #0x58\n"
+      "ldr d13, [x14], #0x8\n"
+      "mov x20, #0x58\n"
       "ldr d19, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "ldr d31, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v13.s }[2], [x13]\n"
+      "ld1 { v13.s }[2], [x14]\n"
       "ld1 { v19.s }[2], [x23]\n"
       "ld1 { v25.s }[2], [x22]\n"
       "ld1 { v31.s }[2], [x21]\n"
       "b 138f\n"
       "127:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x16, #0, 138f\n"
-      "ldr s13, [x13, #0x0]\n"
+      "ldr s13, [x14, #0x0]\n"
       "ldr s19, [x23, #0x0]\n"
       "ldr s25, [x22, #0x0]\n"
       "ldr s31, [x21, #0x0]\n"
       "b 138f\n"
       "128:"  // Height 4: Partial accumulate: partial_2_16
       "tbz x16, #1, 129f\n"
-      "ldr d12, [x13], #0x8\n"
+      "ldr d12, [x14], #0x8\n"
+      "mov x20, #0x48\n"
       "ldr d18, [x23], #0x8\n"
-      "mov x19, #0x48\n"
       "ldr d24, [x22], #0x8\n"
       "ldr d30, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v12.s }[2], [x13]\n"
+      "ld1 { v12.s }[2], [x14]\n"
       "ld1 { v18.s }[2], [x23]\n"
       "ld1 { v24.s }[2], [x22]\n"
       "ld1 { v30.s }[2], [x21]\n"
       "b 138f\n"
       "129:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x16, #0, 138f\n"
-      "ldr s12, [x13, #0x0]\n"
+      "ldr s12, [x14, #0x0]\n"
       "ldr s18, [x23, #0x0]\n"
       "ldr s24, [x22, #0x0]\n"
       "ldr s30, [x21, #0x0]\n"
       "b 138f\n"
       "130:"  // Height 4: Partial accumulate: partial_8_0
       "tbz x16, #3, 134f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x13], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x14], #0x10\n"
       "ld1 { v15.4s }, [x23], #0x10\n"
       "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
       "ld1 { v27.4s }, [x21], #0x10\n"
       "tbz x16, #2, 132f\n"
-      "ld1 { v10.4s }, [x13], #0x10\n"
+      "ld1 { v10.4s }, [x14], #0x10\n"
       "ld1 { v16.4s }, [x23], #0x10\n"
       "ld1 { v22.4s }, [x22], #0x10\n"
       "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x16, #1, 131f\n"
-      "ldr d11, [x13], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x14], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d17, [x23], #0x8\n"
       "ldr d23, [x22], #0x8\n"
       "ldr d29, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v11.s }[2], [x13]\n"
+      "ld1 { v11.s }[2], [x14]\n"
       "ld1 { v17.s }[2], [x23]\n"
       "ld1 { v23.s }[2], [x22]\n"
       "ld1 { v29.s }[2], [x21]\n"
       "b 138f\n"
       "131:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x16, #0, 138f\n"
-      "ldr s11, [x13, #0x0]\n"
+      "ldr s11, [x14, #0x0]\n"
       "ldr s17, [x23, #0x0]\n"
       "ldr s23, [x22, #0x0]\n"
       "ldr s29, [x21, #0x0]\n"
       "b 138f\n"
       "132:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x16, #1, 133f\n"
-      "ldr d10, [x13], #0x8\n"
+      "ldr d10, [x14], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d16, [x23], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d22, [x22], #0x8\n"
       "ldr d28, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v10.s }[2], [x13]\n"
+      "ld1 { v10.s }[2], [x14]\n"
       "ld1 { v16.s }[2], [x23]\n"
       "ld1 { v22.s }[2], [x22]\n"
       "ld1 { v28.s }[2], [x21]\n"
       "b 138f\n"
       "133:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x16, #0, 138f\n"
-      "ldr s10, [x13, #0x0]\n"
+      "ldr s10, [x14, #0x0]\n"
       "ldr s16, [x23, #0x0]\n"
       "ldr s22, [x22, #0x0]\n"
       "ldr s28, [x21, #0x0]\n"
       "b 138f\n"
       "134:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x16, #2, 136f\n"
-      "ld1 { v8.4s }, [x13], #0x10\n"
+      "ld1 { v8.4s }, [x14], #0x10\n"
       "ld1 { v14.4s }, [x23], #0x10\n"
       "ld1 { v20.4s }, [x22], #0x10\n"
       "ld1 { v26.4s }, [x21], #0x10\n"
       "tbz x16, #1, 135f\n"
-      "ldr d9, [x13], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x14], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d15, [x23], #0x8\n"
       "ldr d21, [x22], #0x8\n"
       "ldr d27, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v9.s }[2], [x13]\n"
+      "ld1 { v9.s }[2], [x14]\n"
       "ld1 { v15.s }[2], [x23]\n"
       "ld1 { v21.s }[2], [x22]\n"
       "ld1 { v27.s }[2], [x21]\n"
       "b 138f\n"
       "135:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x16, #0, 138f\n"
-      "ldr s9, [x13, #0x0]\n"
+      "ldr s9, [x14, #0x0]\n"
       "ldr s15, [x23, #0x0]\n"
       "ldr s21, [x22, #0x0]\n"
       "ldr s27, [x21, #0x0]\n"
       "b 138f\n"
       "136:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x16, #1, 137f\n"
-      "ldr d8, [x13], #0x8\n"
+      "ldr d8, [x14], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d14, [x23], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d20, [x22], #0x8\n"
       "ldr d26, [x21], #0x8\n"
       "tbz x16, #0, 138f\n"
-      "ld1 { v8.s }[2], [x13]\n"
+      "ld1 { v8.s }[2], [x14]\n"
       "ld1 { v14.s }[2], [x23]\n"
       "ld1 { v20.s }[2], [x22]\n"
       "ld1 { v26.s }[2], [x21]\n"
       "b 138f\n"
       "137:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x13, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x14, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s14, [x23, #0x0]\n"
       "ldr s20, [x22, #0x0]\n"
       "ldr s26, [x21, #0x0]\n"
       "138:"  // Height 4: Partial accumulate: Done
-      "sub x13, x13, x19\n"
+      "sub x14, x14, x20\n"
       "b 141f\n"
       "139:"  // Height 4: full accumulate
-      "ldr q8, [x13, #0x0]\n"
-      "ldr q9, [x13, #0x10]\n"
-      "ldr q10, [x13, #0x20]\n"
-      "ldr q11, [x13, #0x30]\n"
-      "ldr q12, [x13, #0x40]\n"
-      "ldr q13, [x13, #0x50]\n"
+      "ldr q8, [x14, #0x0]\n"
+      "ldr q9, [x14, #0x10]\n"
+      "ldr q10, [x14, #0x20]\n"
+      "ldr q11, [x14, #0x30]\n"
+      "ldr q12, [x14, #0x40]\n"
+      "ldr q13, [x14, #0x50]\n"
       "ldr q14, [x23, #0x0]\n"
       "ldr q15, [x23, #0x10]\n"
       "ldr q16, [x23, #0x20]\n"
@@ -2135,264 +2135,264 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "141:"  // Height 4: setup done
-      "mov x12, #0x0\n"
+      "mov x13, #0x0\n"
       "142:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr w12, [x20, x13, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 143f\n"
-      "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x10, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x22, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x12, 144f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x10, x10, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x11, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x23, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x13, 144f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 144f\n"
       "143:"  // Height 4: setup direct input
-      "mov x10, %x[input_ptr]\n"
-      "add x24, x10, x19, LSL #2\n"
-      "add x22, x24, x19, LSL #2\n"
-      "add x20, x22, x19, LSL #2\n"
+      "mov x11, %x[input_ptr]\n"
+      "add x25, x11, x20, LSL #2\n"
+      "add x23, x25, x20, LSL #2\n"
+      "add x21, x23, x20, LSL #2\n"
       "144:"  // Height 4: input setup done
-      "cmp x11, #0x4\n"
+      "cmp x12, #0x4\n"
       "blt 147f\n"
-      "ldr q0, [x10, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x11, #0x8\n"
-      "ldr q2, [x22, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
+      "ldr q0, [x11, #0x0]\n"
+      "cmp x12, #0x8\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
       "ldr q4, [x15, #0x0]\n"
+      "ldr q5, [x15, #0x10]\n"
+      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x30]\n"
       "blt 146f\n"
       "145:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr d5, [x15, #0x10]\n"
+      "ldr x10, [x15, #0x48]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr x9, [x15, #0x18]\n"
+      "ldr x9, [x15, #0x58]\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr d6, [x15, #0x20]\n"
+      "ldr x28, [x15, #0x68]\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
-      "ldr x28, [x15, #0x28]\n"
-      "mov v5.d[1], x9\n"
-      "ldr d7, [x15, #0x30]\n"
-      "ldr x27, [x15, #0x38]\n"
-      "add x10, x10, #0x10\n"
+      "ldr d4, [x15, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x27, [x15, #0x78]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr d4, [x15, #0x40]\n"
+      "mov v4.d[1], x10\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x10, [x15, #0x88]\n"
       "fmla v27.4s, v5.4s, v3.s[0]\n"
-      "ldr x26, [x15, #0x48]\n"
+      "ldr d5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
+      "mov v5.d[1], x9\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "ldr d5, [x15, #0x50]\n"
+      "ldr x9, [x15, #0x98]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
-      "mov v4.d[1], x26\n"
+      "add x11, x11, #0x10\n"
       "fmla v28.4s, v6.4s, v3.s[0]\n"
-      "ldr x9, [x15, #0x58]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr d6, [x15, #0x60]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "mov v6.d[1], x28\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
-      "ldr x28, [x15, #0x68]\n"
+      "ldr x28, [x15, #0xa8]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
-      "mov v5.d[1], x9\n"
+      "add x25, x25, #0x10\n"
       "fmla v29.4s, v7.4s, v3.s[0]\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v7.d[1], x27\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "mov v6.d[1], x28\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
-      "ldr x27, [x15, #0x78]\n"
+      "ldr x27, [x15, #0xb8]\n"
       "fmla v24.4s, v4.4s, v2.s[0]\n"
-      "ldr x26, [x15, #0x88]\n"
+      "add x23, x23, #0x10\n"
       "fmla v30.4s, v4.4s, v3.s[0]\n"
       "ldr d4, [x15, #0x80]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
-      "ldr x9, [x15, #0x98]\n"
+      "ldr x10, [x15, #0xc8]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
-      "mov v4.d[1], x26\n"
+      "add x21, x21, #0x10\n"
       "fmla v31.4s, v5.4s, v3.s[0]\n"
       "ldr d5, [x15, #0x90]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr x28, [x15, #0xa8]\n"
+      "mov v5.d[1], x9\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr x27, [x15, #0xb8]\n"
+      "ldr x9, [x15, #0xd8]\n"
       "fmla v20.4s, v6.4s, v2.s[1]\n"
-      "mov v5.d[1], x9\n"
+      "ldr x26, [x11, #0x8]\n"
       "fmla v26.4s, v6.4s, v3.s[1]\n"
       "ldr d6, [x15, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr x26, [x15, #0xc8]\n"
+      "mov v6.d[1], x28\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr x9, [x15, #0xd8]\n"
+      "ldr x28, [x15, #0xe8]\n"
       "fmla v21.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x28\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v27.4s, v7.4s, v3.s[1]\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x27\n"
       "fmla v10.4s, v4.4s, v0.s[1]\n"
-      "ldr x28, [x15, #0xe8]\n"
       "fmla v16.4s, v4.4s, v1.s[1]\n"
-      "ldr x25, [x10, #0x8]\n"
+      "ldr x27, [x15, #0xf8]\n"
       "fmla v22.4s, v4.4s, v2.s[1]\n"
-      "mov v7.d[1], x27\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v28.4s, v4.4s, v3.s[1]\n"
       "ldr d4, [x15, #0xc0]\n"
       "fmla v11.4s, v5.4s, v0.s[1]\n"
-      "ldr x27, [x15, #0xf8]\n"
+      "mov v4.d[1], x10\n"
       "fmla v17.4s, v5.4s, v1.s[1]\n"
-      "add x24, x24, #0x10\n"
+      "ldr x10, [x15, #0x108]\n"
       "fmla v23.4s, v5.4s, v2.s[1]\n"
-      "mov v4.d[1], x26\n"
+      "ldr x20, [x21, #0x8]\n"
       "fmla v29.4s, v5.4s, v3.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "fmla v12.4s, v6.4s, v0.s[1]\n"
       "ldr d5, [x15, #0xd0]\n"
+      "fmla v12.4s, v6.4s, v0.s[1]\n"
+      "mov v5.d[1], x9\n"
       "fmla v18.4s, v6.4s, v1.s[1]\n"
-      "ldr x26, [x15, #0x108]\n"
+      "ldr x9, [x15, #0x118]\n"
       "fmla v24.4s, v6.4s, v2.s[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "sub x12, x12, #0x4\n"
       "fmla v30.4s, v6.4s, v3.s[1]\n"
-      "mov v5.d[1], x9\n"
-      "fmla v13.4s, v7.4s, v0.s[1]\n"
       "ldr d6, [x15, #0xe0]\n"
+      "fmla v13.4s, v7.4s, v0.s[1]\n"
+      "mov v6.d[1], x28\n"
       "fmla v19.4s, v7.4s, v1.s[1]\n"
-      "ldr x9, [x15, #0x118]\n"
+      "ldr x28, [x15, #0x128]\n"
       "fmla v25.4s, v7.4s, v2.s[1]\n"
-      "add x22, x22, #0x10\n"
+      "cmp x12, #0x8\n"
       "fmla v31.4s, v7.4s, v3.s[1]\n"
-      "mov v6.d[1], x28\n"
+      "ldr d7, [x15, #0xf0]\n"
+      "mov v7.d[1], x27\n"
       "fmla v8.4s, v4.4s, v0.s[2]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v14.4s, v4.4s, v1.s[2]\n"
-      "ldr d7, [x15, #0xf0]\n"
+      "ldr x27, [x15, #0x138]\n"
       "fmla v20.4s, v4.4s, v2.s[2]\n"
-      "ldr x28, [x15, #0x128]\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v26.4s, v4.4s, v3.s[2]\n"
       "ldr d4, [x15, #0x100]\n"
       "fmla v9.4s, v5.4s, v0.s[2]\n"
-      "mov v7.d[1], x27\n"
+      "mov v4.d[1], x10\n"
       "fmla v15.4s, v5.4s, v1.s[2]\n"
-      "ldr x27, [x15, #0x138]\n"
+      "ldr x10, [x15, #0x148]\n"
       "fmla v21.4s, v5.4s, v2.s[2]\n"
-      "mov v4.d[1], x26\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v27.4s, v5.4s, v3.s[2]\n"
       "ldr d5, [x15, #0x110]\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr x26, [x15, #0x148]\n"
+      "mov v5.d[1], x9\n"
       "fmla v16.4s, v6.4s, v1.s[2]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x9, [x15, #0x158]\n"
       "fmla v22.4s, v6.4s, v2.s[2]\n"
-      "mov v5.d[1], x9\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v28.4s, v6.4s, v3.s[2]\n"
       "ldr d6, [x15, #0x120]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr x9, [x15, #0x158]\n"
+      "mov v6.d[1], x28\n"
       "fmla v17.4s, v7.4s, v1.s[2]\n"
-      "add x20, x20, #0x10\n"
+      "ldr x28, [x15, #0x168]\n"
       "fmla v23.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x28\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v29.4s, v7.4s, v3.s[2]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "fmla v12.4s, v4.4s, v0.s[2]\n"
       "ldr d7, [x15, #0x130]\n"
+      "mov v7.d[1], x27\n"
+      "fmla v12.4s, v4.4s, v0.s[2]\n"
       "fmla v18.4s, v4.4s, v1.s[2]\n"
-      "ldr x28, [x15, #0x168]\n"
+      "ldr x27, [x15, #0x178]\n"
       "fmla v24.4s, v4.4s, v2.s[2]\n"
-      "ldr x19, [x20, #0x8]\n"
       "fmla v30.4s, v4.4s, v3.s[2]\n"
-      "mov v7.d[1], x27\n"
-      "fmla v13.4s, v5.4s, v0.s[2]\n"
       "ldr d4, [x15, #0x140]\n"
+      "fmla v13.4s, v5.4s, v0.s[2]\n"
+      "mov v4.d[1], x10\n"
       "fmla v19.4s, v5.4s, v1.s[2]\n"
-      "ldr x27, [x15, #0x178]\n"
       "fmla v25.4s, v5.4s, v2.s[2]\n"
-      "sub x11, x11, #0x4\n"
       "fmla v31.4s, v5.4s, v3.s[2]\n"
-      "mov v4.d[1], x26\n"
-      "fmla v8.4s, v6.4s, v0.s[3]\n"
       "ldr d5, [x15, #0x150]\n"
+      "fmla v8.4s, v6.4s, v0.s[3]\n"
+      "mov v5.d[1], x9\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "cmp x11, #0x8\n"
       "fmla v20.4s, v6.4s, v2.s[3]\n"
       "fmla v26.4s, v6.4s, v3.s[3]\n"
-      "mov v5.d[1], x9\n"
-      "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr d6, [x15, #0x160]\n"
+      "fmla v9.4s, v7.4s, v0.s[3]\n"
+      "mov v6.d[1], x28\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "fmla v21.4s, v7.4s, v2.s[3]\n"
       "fmla v27.4s, v7.4s, v3.s[3]\n"
-      "mov v6.d[1], x28\n"
-      "fmla v10.4s, v4.4s, v0.s[3]\n"
       "ldr d7, [x15, #0x170]\n"
-      "fmla v16.4s, v4.4s, v1.s[3]\n"
+      "mov v7.d[1], x27\n"
       "add x15, x15, #0x180\n"
+      "fmla v10.4s, v4.4s, v0.s[3]\n"
+      "ldr x10, [x15, #0x8]\n"
+      "fmla v16.4s, v4.4s, v1.s[3]\n"
+      "ldr x9, [x15, #0x18]\n"
       "fmla v22.4s, v4.4s, v2.s[3]\n"
-      "ldr x26, [x15, #0x8]\n"
+      "ldr x28, [x15, #0x28]\n"
       "fmla v28.4s, v4.4s, v3.s[3]\n"
-      "mov v7.d[1], x27\n"
-      "fmla v11.4s, v5.4s, v0.s[3]\n"
       "ldr d4, [x15, #0x0]\n"
+      "fmla v11.4s, v5.4s, v0.s[3]\n"
+      "ldr x27, [x15, #0x38]\n"
       "fmla v17.4s, v5.4s, v1.s[3]\n"
+      "mov v4.d[1], x10\n"
       "fmla v23.4s, v5.4s, v2.s[3]\n"
       "fmla v29.4s, v5.4s, v3.s[3]\n"
-      "mov v4.d[1], x26\n"
+      "ldr d5, [x15, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
+      "mov v5.d[1], x9\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
       "fmla v24.4s, v6.4s, v2.s[3]\n"
       "fmla v30.4s, v6.4s, v3.s[3]\n"
+      "ldr d6, [x15, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x10, #0x0]\n"
+      "ldr d0, [x11, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x24, #0x0]\n"
+      "ldr d1, [x25, #0x0]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
-      "ldr d2, [x22, #0x0]\n"
+      "ldr d2, [x23, #0x0]\n"
       "fmla v31.4s, v7.4s, v3.s[3]\n"
-      "mov v0.d[1], x25\n"
-      "mov v1.d[1], x23\n"
-      "ldr d3, [x20, #0x0]\n"
-      "mov v2.d[1], x21\n"
-      "mov v3.d[1], x19\n"
+      "ldr d3, [x21, #0x0]\n"
+      "ldr d7, [x15, #0x30]\n"
+      "mov v6.d[1], x28\n"
+      "mov v0.d[1], x26\n"
+      "mov v1.d[1], x24\n"
+      "mov v2.d[1], x22\n"
+      "mov v3.d[1], x20\n"
+      "mov v7.d[1], x27\n"
       "bge 145b\n"
       "146:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x15, #0x10]\n"
+      "add x11, x11, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x15, #0x20]\n"
+      "add x25, x25, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
+      "add x23, x23, #0x10\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
       "ldr q4, [x15, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "sub x11, x11, #0x4\n"
+      "add x21, x21, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "add x10, x10, #0x10\n"
+      "sub x12, x12, #0x4\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v27.4s, v5.4s, v3.s[0]\n"
       "ldr q5, [x15, #0x50]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v28.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x15, #0x60]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
-      "add x20, x20, #0x10\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v29.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x15, #0x70]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
@@ -2493,20 +2493,20 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmla v25.4s, v7.4s, v2.s[3]\n"
       "fmla v31.4s, v7.4s, v3.s[3]\n"
       "147:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x11, 149f\n"
+      "cbz x12, 149f\n"
       "148:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x10], #0x4\n"
-      "sub x11, x11, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x11], #0x4\n"
+      "sub x12, x12, #0x1\n"
+      "ldr s1, [x25], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "ldr q4, [x15, #0x0]\n"
-      "ldr q5, [x15, #0x10]\n"
-      "ldr q6, [x15, #0x20]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q7, [x15, #0x30]\n"
+      "ldr q5, [x15, #0x10]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
+      "ldr q6, [x15, #0x20]\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
+      "ldr q7, [x15, #0x30]\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
       "ldr q4, [x15, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
@@ -2531,25 +2531,23 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
       "fmla v31.4s, v5.4s, v3.s[0]\n"
-      "cbnz x11, 148b\n"
+      "cbnz x12, 148b\n"
       "149:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x12, x12, #0x1\n"
-      "cmp x12, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x13, x13, #0x1\n"
+      "cmp x13, x20\n"
       "bne 142b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x13, #0x0]\n"
-      "add x23, x13, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x14, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 150f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -2560,16 +2558,6 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
       "fmin v20.4s, v20.4s, v0.4s\n"
@@ -2580,32 +2568,44 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "fmin v25.4s, v25.4s, v0.4s\n"
       "fmin v26.4s, v26.4s, v0.4s\n"
       "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
       "fmin v28.4s, v28.4s, v0.4s\n"
       "fmin v29.4s, v29.4s, v0.4s\n"
       "fmin v30.4s, v30.4s, v0.4s\n"
       "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v31.4s, v31.4s, v0.4s\n"
       "150:"  // Height 4: No activation
       "cmp x16, #0x18\n"
       "bge 163f\n"
       "tbz x16, #4, 154f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
-      "st1 { v11.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
+      "st1 { v11.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
@@ -2619,51 +2619,51 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "st1 { v28.4s }, [x21], #0x10\n"
       "st1 { v29.4s }, [x21], #0x10\n"
       "tbz x16, #2, 152f\n"
-      "st1 { v12.4s }, [x13], #0x10\n"
+      "st1 { v12.4s }, [x14], #0x10\n"
       "st1 { v18.4s }, [x23], #0x10\n"
       "st1 { v24.4s }, [x22], #0x10\n"
       "st1 { v30.4s }, [x21], #0x10\n"
       "tbz x16, #1, 151f\n"
-      "str d13, [x13], #0x8\n"
+      "str d13, [x14], #0x8\n"
       "str d19, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "str d31, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v13.s }[2], [x13]\n"
+      "st1 { v13.s }[2], [x14]\n"
       "st1 { v19.s }[2], [x23]\n"
       "st1 { v25.s }[2], [x22]\n"
       "st1 { v31.s }[2], [x21]\n"
       "b 162f\n"
       "151:"  // Height 4: Partial direct writeback: partial_1_20
       "tbz x16, #0, 162f\n"
-      "str s13, [x13, #0x0]\n"
+      "str s13, [x14, #0x0]\n"
       "str s19, [x23, #0x0]\n"
       "str s25, [x22, #0x0]\n"
       "str s31, [x21, #0x0]\n"
       "b 162f\n"
       "152:"  // Height 4: Partial direct writeback: partial_2_16
       "tbz x16, #1, 153f\n"
-      "str d12, [x13], #0x8\n"
+      "str d12, [x14], #0x8\n"
       "str d18, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "str d30, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v12.s }[2], [x13]\n"
+      "st1 { v12.s }[2], [x14]\n"
       "st1 { v18.s }[2], [x23]\n"
       "st1 { v24.s }[2], [x22]\n"
       "st1 { v30.s }[2], [x21]\n"
       "b 162f\n"
       "153:"  // Height 4: Partial direct writeback: partial_1_16
       "tbz x16, #0, 162f\n"
-      "str s12, [x13, #0x0]\n"
+      "str s12, [x14, #0x0]\n"
       "str s18, [x23, #0x0]\n"
       "str s24, [x22, #0x0]\n"
       "str s30, [x21, #0x0]\n"
       "b 162f\n"
       "154:"  // Height 4: Partial direct writeback: partial_8_0
       "tbz x16, #3, 158f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
-      "st1 { v9.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
+      "st1 { v9.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v15.4s }, [x23], #0x10\n"
       "st1 { v20.4s }, [x22], #0x10\n"
@@ -2671,98 +2671,98 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "st1 { v26.4s }, [x21], #0x10\n"
       "st1 { v27.4s }, [x21], #0x10\n"
       "tbz x16, #2, 156f\n"
-      "st1 { v10.4s }, [x13], #0x10\n"
+      "st1 { v10.4s }, [x14], #0x10\n"
       "st1 { v16.4s }, [x23], #0x10\n"
       "st1 { v22.4s }, [x22], #0x10\n"
       "st1 { v28.4s }, [x21], #0x10\n"
       "tbz x16, #1, 155f\n"
-      "str d11, [x13], #0x8\n"
+      "str d11, [x14], #0x8\n"
       "str d17, [x23], #0x8\n"
       "str d23, [x22], #0x8\n"
       "str d29, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v11.s }[2], [x13]\n"
+      "st1 { v11.s }[2], [x14]\n"
       "st1 { v17.s }[2], [x23]\n"
       "st1 { v23.s }[2], [x22]\n"
       "st1 { v29.s }[2], [x21]\n"
       "b 162f\n"
       "155:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x16, #0, 162f\n"
-      "str s11, [x13, #0x0]\n"
+      "str s11, [x14, #0x0]\n"
       "str s17, [x23, #0x0]\n"
       "str s23, [x22, #0x0]\n"
       "str s29, [x21, #0x0]\n"
       "b 162f\n"
       "156:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x16, #1, 157f\n"
-      "str d10, [x13], #0x8\n"
+      "str d10, [x14], #0x8\n"
       "str d16, [x23], #0x8\n"
       "str d22, [x22], #0x8\n"
       "str d28, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v10.s }[2], [x13]\n"
+      "st1 { v10.s }[2], [x14]\n"
       "st1 { v16.s }[2], [x23]\n"
       "st1 { v22.s }[2], [x22]\n"
       "st1 { v28.s }[2], [x21]\n"
       "b 162f\n"
       "157:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x16, #0, 162f\n"
-      "str s10, [x13, #0x0]\n"
+      "str s10, [x14, #0x0]\n"
       "str s16, [x23, #0x0]\n"
       "str s22, [x22, #0x0]\n"
       "str s28, [x21, #0x0]\n"
       "b 162f\n"
       "158:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x16, #2, 160f\n"
-      "st1 { v8.4s }, [x13], #0x10\n"
+      "st1 { v8.4s }, [x14], #0x10\n"
       "st1 { v14.4s }, [x23], #0x10\n"
       "st1 { v20.4s }, [x22], #0x10\n"
       "st1 { v26.4s }, [x21], #0x10\n"
       "tbz x16, #1, 159f\n"
-      "str d9, [x13], #0x8\n"
+      "str d9, [x14], #0x8\n"
       "str d15, [x23], #0x8\n"
       "str d21, [x22], #0x8\n"
       "str d27, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v9.s }[2], [x13]\n"
+      "st1 { v9.s }[2], [x14]\n"
       "st1 { v15.s }[2], [x23]\n"
       "st1 { v21.s }[2], [x22]\n"
       "st1 { v27.s }[2], [x21]\n"
       "b 162f\n"
       "159:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x16, #0, 162f\n"
-      "str s9, [x13, #0x0]\n"
+      "str s9, [x14, #0x0]\n"
       "str s15, [x23, #0x0]\n"
       "str s21, [x22, #0x0]\n"
       "str s27, [x21, #0x0]\n"
       "b 162f\n"
       "160:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x16, #1, 161f\n"
-      "str d8, [x13], #0x8\n"
+      "str d8, [x14], #0x8\n"
       "str d14, [x23], #0x8\n"
       "str d20, [x22], #0x8\n"
       "str d26, [x21], #0x8\n"
       "tbz x16, #0, 162f\n"
-      "st1 { v8.s }[2], [x13]\n"
+      "st1 { v8.s }[2], [x14]\n"
       "st1 { v14.s }[2], [x23]\n"
       "st1 { v20.s }[2], [x22]\n"
       "st1 { v26.s }[2], [x21]\n"
       "b 162f\n"
       "161:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x13, #0x0]\n"
+      "str s8, [x14, #0x0]\n"
       "str s14, [x23, #0x0]\n"
       "str s20, [x22, #0x0]\n"
       "str s26, [x21, #0x0]\n"
       "162:"  // Height 4: Partial direct writeback: Done
       "b 164f\n"
       "163:"  // Height 4: Full writeback
-      "str q8, [x13, #0x0]\n"
-      "str q9, [x13, #0x10]\n"
-      "str q10, [x13, #0x20]\n"
-      "str q11, [x13, #0x30]\n"
-      "str q12, [x13, #0x40]\n"
-      "str q13, [x13, #0x50]\n"
-      "add x13, x13, #0x60\n"
+      "str q8, [x14, #0x0]\n"
+      "str q9, [x14, #0x10]\n"
+      "str q10, [x14, #0x20]\n"
+      "str q11, [x14, #0x30]\n"
+      "str q12, [x14, #0x40]\n"
+      "str q13, [x14, #0x50]\n"
+      "add x14, x14, #0x60\n"
       "str q14, [x23, #0x0]\n"
       "str q15, [x23, #0x10]\n"
       "str q16, [x23, #0x20]\n"
@@ -2786,20 +2786,20 @@ void a64_hybrid_fp32_mla_4x24_a55 (
       "bgt 125b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 166f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 165f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "165:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "166:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp
index 37d59cc327..5fb71c95b7 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -99,117 +99,117 @@ void a64_hybrid_fp32_mla_4x24 (
       "cmp %x[M], #0x2\n"
       "bgt 83f\n"
       "beq 42f\n"
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[bias]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x27, 3f\n"
-      "ldr q8, [x27, #0x0]\n"
-      "ldr q9, [x27, #0x10]\n"
-      "ldr q10, [x27, #0x20]\n"
-      "ldr q11, [x27, #0x30]\n"
-      "ldr q12, [x27, #0x40]\n"
-      "ldr q13, [x27, #0x50]\n"
-      "add x27, x27, #0x60\n"
+      "cbz x10, 3f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
+      "add x10, x10, #0x60\n"
       "b 18f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 17f\n"
       "cmp x9, #0x18\n"
       "bge 16f\n"
       "tbz x9, #4, 7f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
       "tbz x9, #2, 5f\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
       "tbz x9, #1, 4f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x26], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "mov x20, #0x58\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v13.s }[2], [x26]\n"
+      "ld1 { v13.s }[2], [x27]\n"
       "b 15f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 15f\n"
-      "ldr s13, [x26, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
       "b 15f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_16
       "tbz x9, #1, 6f\n"
-      "ldr d12, [x26], #0x8\n"
-      "mov x19, #0x48\n"
+      "ldr d12, [x27], #0x8\n"
+      "mov x20, #0x48\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v12.s }[2], [x26]\n"
+      "ld1 { v12.s }[2], [x27]\n"
       "b 15f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 15f\n"
-      "ldr s12, [x26, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
       "b 15f\n"
       "7:"  // Height 1: Partial accumulate: partial_8_0
       "tbz x9, #3, 11f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
       "tbz x9, #2, 9f\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
       "tbz x9, #1, 8f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x26], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v11.s }[2], [x26]\n"
+      "ld1 { v11.s }[2], [x27]\n"
       "b 15f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 15f\n"
-      "ldr s11, [x26, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
       "b 15f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x9, #1, 10f\n"
-      "ldr d10, [x26], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x27], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v10.s }[2], [x26]\n"
+      "ld1 { v10.s }[2], [x27]\n"
       "b 15f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 15f\n"
-      "ldr s10, [x26, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
       "b 15f\n"
       "11:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x9, #2, 13f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
       "tbz x9, #1, 12f\n"
-      "ldr d9, [x26], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x27], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v9.s }[2], [x26]\n"
+      "ld1 { v9.s }[2], [x27]\n"
       "b 15f\n"
       "12:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 15f\n"
-      "ldr s9, [x26, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
       "b 15f\n"
       "13:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x9, #1, 14f\n"
-      "ldr d8, [x26], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x27], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v8.s }[2], [x26]\n"
+      "ld1 { v8.s }[2], [x27]\n"
       "b 15f\n"
       "14:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
       "15:"  // Height 1: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 18f\n"
       "16:"  // Height 1: full accumulate
-      "ldr q8, [x26, #0x0]\n"
-      "ldr q9, [x26, #0x10]\n"
-      "ldr q10, [x26, #0x20]\n"
-      "ldr q11, [x26, #0x30]\n"
-      "ldr q12, [x26, #0x40]\n"
-      "ldr q13, [x26, #0x50]\n"
+      "ldr q8, [x27, #0x0]\n"
+      "ldr q9, [x27, #0x10]\n"
+      "ldr q10, [x27, #0x20]\n"
+      "ldr q11, [x27, #0x30]\n"
+      "ldr q12, [x27, #0x40]\n"
+      "ldr q13, [x27, #0x50]\n"
       "b 18f\n"
       "17:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -219,61 +219,57 @@ void a64_hybrid_fp32_mla_4x24 (
       "movi v12.16b, #0x0\n"
       "movi v13.16b, #0x0\n"
       "18:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "19:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "21:"  // Height 1: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 24f\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
-      "cmp x24, #0x8\n"
-      "blt 23f\n"
-      "22:"  // Height 1: Multiply loop: Main loop head
-      "fmla v8.4s, v4.4s, v0.s[0]\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q6, [x28, #0x20]\n"
-      "sub x24, x24, #0x4\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q7, [x28, #0x30]\n"
-      "cmp x24, #0x8\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "blt 23f\n"
+      "22:"  // Height 1: Multiply loop: Main loop head
+      "fmla v8.4s, v4.4s, v0.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
-      "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
+      "fmla v12.4s, v4.4s, v0.s[0]\n"
       "ldr q4, [x28, #0x80]\n"
-      "fmla v8.4s, v6.4s, v0.s[1]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x90]\n"
+      "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x28, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "fmla v10.4s, v4.4s, v0.s[1]\n"
       "ldr q7, [x28, #0xb0]\n"
+      "fmla v10.4s, v4.4s, v0.s[1]\n"
       "ldr q4, [x28, #0xc0]\n"
       "fmla v11.4s, v5.4s, v0.s[1]\n"
       "ldr q5, [x28, #0xd0]\n"
       "fmla v12.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x28, #0xe0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v13.4s, v7.4s, v0.s[1]\n"
-      "fmla v8.4s, v4.4s, v0.s[2]\n"
       "ldr q7, [x28, #0xf0]\n"
+      "fmla v8.4s, v4.4s, v0.s[2]\n"
       "ldr q4, [x28, #0x100]\n"
       "fmla v9.4s, v5.4s, v0.s[2]\n"
       "ldr q5, [x28, #0x110]\n"
@@ -289,44 +285,45 @@ void a64_hybrid_fp32_mla_4x24 (
       "ldr q6, [x28, #0x160]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x28, #0x170]\n"
-      "add x28, x28, #0x180\n"
+      "sub x25, x25, #0x4\n"
+      "add x24, x24, #0x10\n"
       "fmla v10.4s, v4.4s, v0.s[3]\n"
-      "ldr q4, [x28, #0x0]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
+      "cmp x25, #0x8\n"
+      "add x28, x28, #0x180\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
+      "ldr q6, [x28, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "bge 22b\n"
       "23:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
       "ldr q4, [x28, #0x40]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
-      "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
-      "fmla v13.4s, v5.4s, v0.s[0]\n"
+      "fmla v12.4s, v4.4s, v0.s[0]\n"
       "ldr q4, [x28, #0x80]\n"
+      "fmla v13.4s, v5.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x90]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr q6, [x28, #0xa0]\n"
+      "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x28, #0xb0]\n"
       "fmla v10.4s, v4.4s, v0.s[1]\n"
       "ldr q4, [x28, #0xc0]\n"
       "fmla v11.4s, v5.4s, v0.s[1]\n"
       "ldr q5, [x28, #0xd0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v12.4s, v6.4s, v0.s[1]\n"
-      "fmla v13.4s, v7.4s, v0.s[1]\n"
       "ldr q6, [x28, #0xe0]\n"
+      "fmla v13.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x28, #0xf0]\n"
       "fmla v8.4s, v4.4s, v0.s[2]\n"
       "ldr q4, [x28, #0x100]\n"
@@ -344,294 +341,297 @@ void a64_hybrid_fp32_mla_4x24 (
       "ldr q6, [x28, #0x160]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x28, #0x170]\n"
-      "add x28, x28, #0x180\n"
+      "add x24, x24, #0x10\n"
+      "sub x25, x25, #0x4\n"
       "fmla v10.4s, v4.4s, v0.s[3]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x28, x28, #0x180\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
       "24:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 26f\n"
+      "cbz x25, 26f\n"
       "25:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "sub x24, x24, #0x1\n"
+      "ldr s0, [x24], #0x4\n"
       "ldr q4, [x28, #0x0]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
+      "sub x25, x25, #0x1\n"
       "ldr q5, [x28, #0x10]\n"
       "ldr q6, [x28, #0x20]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x28, #0x30]\n"
       "ldr q4, [x28, #0x40]\n"
-      "ldr q5, [x28, #0x50]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x28, x28, #0x60\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
+      "ldr q5, [x28, #0x50]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
-      "cbnz x24, 25b\n"
+      "add x28, x28, #0x60\n"
+      "cbnz x25, 25b\n"
       "26:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 19b\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
       "27:"  // Height 1: No activation
       "cmp x9, #0x18\n"
       "bge 40f\n"
       "tbz x9, #4, 31f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v11.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v11.4s }, [x27], #0x10\n"
       "tbz x9, #2, 29f\n"
-      "st1 { v12.4s }, [x26], #0x10\n"
+      "st1 { v12.4s }, [x27], #0x10\n"
       "tbz x9, #1, 28f\n"
-      "str d13, [x26], #0x8\n"
+      "str d13, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v13.s }[2], [x26]\n"
+      "st1 { v13.s }[2], [x27]\n"
       "b 39f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_20
       "tbz x9, #0, 39f\n"
-      "str s13, [x26, #0x0]\n"
+      "str s13, [x27, #0x0]\n"
       "b 39f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_16
       "tbz x9, #1, 30f\n"
-      "str d12, [x26], #0x8\n"
+      "str d12, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v12.s }[2], [x26]\n"
+      "st1 { v12.s }[2], [x27]\n"
       "b 39f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_16
       "tbz x9, #0, 39f\n"
-      "str s12, [x26, #0x0]\n"
+      "str s12, [x27, #0x0]\n"
       "b 39f\n"
       "31:"  // Height 1: Partial direct writeback: partial_8_0
       "tbz x9, #3, 35f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
       "tbz x9, #2, 33f\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
       "tbz x9, #1, 32f\n"
-      "str d11, [x26], #0x8\n"
+      "str d11, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v11.s }[2], [x26]\n"
+      "st1 { v11.s }[2], [x27]\n"
       "b 39f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 39f\n"
-      "str s11, [x26, #0x0]\n"
+      "str s11, [x27, #0x0]\n"
       "b 39f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 34f\n"
-      "str d10, [x26], #0x8\n"
+      "str d10, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v10.s }[2], [x26]\n"
+      "st1 { v10.s }[2], [x27]\n"
       "b 39f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 39f\n"
-      "str s10, [x26, #0x0]\n"
+      "str s10, [x27, #0x0]\n"
       "b 39f\n"
       "35:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 37f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
       "tbz x9, #1, 36f\n"
-      "str d9, [x26], #0x8\n"
+      "str d9, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v9.s }[2], [x26]\n"
+      "st1 { v9.s }[2], [x27]\n"
       "b 39f\n"
       "36:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 39f\n"
-      "str s9, [x26, #0x0]\n"
+      "str s9, [x27, #0x0]\n"
       "b 39f\n"
       "37:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 38f\n"
-      "str d8, [x26], #0x8\n"
+      "str d8, [x27], #0x8\n"
       "tbz x9, #0, 39f\n"
-      "st1 { v8.s }[2], [x26]\n"
+      "st1 { v8.s }[2], [x27]\n"
       "b 39f\n"
       "38:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x26, #0x0]\n"
+      "str s8, [x27, #0x0]\n"
       "39:"  // Height 1: Partial direct writeback: Done
       "b 41f\n"
       "40:"  // Height 1: Full writeback
-      "str q8, [x26, #0x0]\n"
-      "str q9, [x26, #0x10]\n"
-      "str q10, [x26, #0x20]\n"
-      "str q11, [x26, #0x30]\n"
-      "str q12, [x26, #0x40]\n"
-      "str q13, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
+      "str q8, [x27, #0x0]\n"
+      "str q9, [x27, #0x10]\n"
+      "str q10, [x27, #0x20]\n"
+      "str q11, [x27, #0x30]\n"
+      "str q12, [x27, #0x40]\n"
+      "str q13, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
       "41:"  // Height 1: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 2b\n"
       "b 166f\n"
       "42:"  // Height 2
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "43:"  // Height 2: Column loop
-      "cbz x27, 44f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 44f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q9, [x27, #0x10]\n"
-      "ldr q10, [x27, #0x20]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q11, [x27, #0x30]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "mov v16.16b, v10.16b\n"
-      "ldr q12, [x27, #0x40]\n"
-      "ldr q13, [x27, #0x50]\n"
       "mov v17.16b, v11.16b\n"
-      "add x27, x27, #0x60\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "mov v18.16b, v12.16b\n"
       "mov v19.16b, v13.16b\n"
+      "add x10, x10, #0x60\n"
       "b 59f\n"
       "44:"  // Height 2: no bias
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
+      "add x23, x27, x20, LSL #2\n"
       "bge 57f\n"
       "tbz x9, #4, 48f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
       "tbz x9, #2, 46f\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
       "tbz x9, #1, 45f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 56f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 56f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 56f\n"
       "46:"  // Height 2: Partial accumulate: partial_2_16
       "tbz x9, #1, 47f\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 56f\n"
       "47:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 56f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 56f\n"
       "48:"  // Height 2: Partial accumulate: partial_8_0
       "tbz x9, #3, 52f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
       "tbz x9, #2, 50f\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
       "tbz x9, #1, 49f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 56f\n"
       "49:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 56f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 56f\n"
       "50:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x9, #1, 51f\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 56f\n"
       "51:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 56f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
       "b 56f\n"
       "52:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x9, #2, 54f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
       "tbz x9, #1, 53f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
       "b 56f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 56f\n"
-      "ldr s9, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
       "b 56f\n"
       "54:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x9, #1, 55f\n"
-      "ldr d8, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x9, #0, 56f\n"
-      "ld1 { v8.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
+      "ld1 { v8.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
       "b 56f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
+      "ldr s8, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
       "56:"  // Height 2: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 59f\n"
       "57:"  // Height 2: full accumulate
-      "ldr q8, [x26, #0x0]\n"
-      "ldr q9, [x26, #0x10]\n"
-      "ldr q10, [x26, #0x20]\n"
-      "ldr q11, [x26, #0x30]\n"
-      "ldr q12, [x26, #0x40]\n"
-      "ldr q13, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
+      "ldr q8, [x27, #0x0]\n"
+      "ldr q9, [x27, #0x10]\n"
+      "ldr q10, [x27, #0x20]\n"
+      "ldr q11, [x27, #0x30]\n"
+      "ldr q12, [x27, #0x40]\n"
+      "ldr q13, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
       "b 59f\n"
       "58:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -647,60 +647,60 @@ void a64_hybrid_fp32_mla_4x24 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "59:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "60:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 62f\n"
       "61:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
       "62:"  // Height 2: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 65f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x8\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x8\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 64f\n"
       "63:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q4, [x28, #0x40]\n"
+      "sub x25, x25, #0x4\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "sub x24, x24, #0x4\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr q4, [x28, #0x40]\n"
-      "cmp x24, #0x8\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "add x24, x24, #0x10\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
+      "cmp x25, #0x8\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
       "ldr q4, [x28, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "ldr q5, [x28, #0x90]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "ldr q6, [x28, #0xa0]\n"
@@ -749,37 +749,37 @@ void a64_hybrid_fp32_mla_4x24 (
       "ldr q4, [x28, #0x0]\n"
       "fmla v11.4s, v5.4s, v0.s[3]\n"
       "fmla v17.4s, v5.4s, v1.s[3]\n"
+      "ldr q5, [x28, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
+      "ldr q6, [x28, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x22, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q7, [x28, #0x30]\n"
       "bge 63b\n"
       "64:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q4, [x28, #0x40]\n"
+      "add x24, x24, #0x10\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "ldr q4, [x28, #0x40]\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "add x23, x23, #0x10\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "sub x25, x25, #0x4\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
       "ldr q4, [x28, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "ldr q5, [x28, #0x90]\n"
@@ -835,383 +835,383 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v13.4s, v7.4s, v0.s[3]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
       "65:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 67f\n"
+      "cbz x25, 67f\n"
       "66:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "sub x24, x24, #0x1\n"
-      "ldr s1, [x22], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "sub x25, x25, #0x1\n"
       "ldr q4, [x28, #0x0]\n"
-      "fmla v8.4s, v4.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x10]\n"
+      "fmla v8.4s, v4.4s, v0.s[0]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
       "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "ldr q4, [x28, #0x40]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
+      "ldr q4, [x28, #0x40]\n"
       "ldr q5, [x28, #0x50]\n"
-      "add x28, x28, #0x60\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
+      "add x28, x28, #0x60\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
-      "cbnz x24, 66b\n"
+      "cbnz x25, 66b\n"
       "67:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 60b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 68f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "68:"  // Height 2: No activation
       "cmp x9, #0x18\n"
       "bge 81f\n"
       "tbz x9, #4, 72f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v11.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v11.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
       "tbz x9, #2, 70f\n"
-      "st1 { v12.4s }, [x26], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v12.4s }, [x27], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
       "tbz x9, #1, 69f\n"
-      "str d13, [x26], #0x8\n"
-      "str d19, [x22], #0x8\n"
+      "str d13, [x27], #0x8\n"
+      "str d19, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v13.s }[2], [x26]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v13.s }[2], [x27]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 80f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_20
       "tbz x9, #0, 80f\n"
-      "str s13, [x26, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "str s13, [x27, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 80f\n"
       "70:"  // Height 2: Partial direct writeback: partial_2_16
       "tbz x9, #1, 71f\n"
-      "str d12, [x26], #0x8\n"
-      "str d18, [x22], #0x8\n"
+      "str d12, [x27], #0x8\n"
+      "str d18, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v12.s }[2], [x26]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v12.s }[2], [x27]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 80f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_16
       "tbz x9, #0, 80f\n"
-      "str s12, [x26, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "str s12, [x27, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 80f\n"
       "72:"  // Height 2: Partial direct writeback: partial_8_0
       "tbz x9, #3, 76f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
       "tbz x9, #2, 74f\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
       "tbz x9, #1, 73f\n"
-      "str d11, [x26], #0x8\n"
-      "str d17, [x22], #0x8\n"
+      "str d11, [x27], #0x8\n"
+      "str d17, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v11.s }[2], [x26]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v11.s }[2], [x27]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 80f\n"
       "73:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 80f\n"
-      "str s11, [x26, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "str s11, [x27, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 80f\n"
       "74:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 75f\n"
-      "str d10, [x26], #0x8\n"
-      "str d16, [x22], #0x8\n"
+      "str d10, [x27], #0x8\n"
+      "str d16, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v10.s }[2], [x26]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v10.s }[2], [x27]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 80f\n"
       "75:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 80f\n"
-      "str s10, [x26, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s10, [x27, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "b 80f\n"
       "76:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 78f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
       "tbz x9, #1, 77f\n"
-      "str d9, [x26], #0x8\n"
-      "str d15, [x22], #0x8\n"
+      "str d9, [x27], #0x8\n"
+      "str d15, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v9.s }[2], [x26]\n"
-      "st1 { v15.s }[2], [x22]\n"
+      "st1 { v9.s }[2], [x27]\n"
+      "st1 { v15.s }[2], [x23]\n"
       "b 80f\n"
       "77:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 80f\n"
-      "str s9, [x26, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
+      "str s9, [x27, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
       "b 80f\n"
       "78:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 79f\n"
-      "str d8, [x26], #0x8\n"
-      "str d14, [x22], #0x8\n"
+      "str d8, [x27], #0x8\n"
+      "str d14, [x23], #0x8\n"
       "tbz x9, #0, 80f\n"
-      "st1 { v8.s }[2], [x26]\n"
-      "st1 { v14.s }[2], [x22]\n"
+      "st1 { v8.s }[2], [x27]\n"
+      "st1 { v14.s }[2], [x23]\n"
       "b 80f\n"
       "79:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x26, #0x0]\n"
-      "str s14, [x22, #0x0]\n"
+      "str s8, [x27, #0x0]\n"
+      "str s14, [x23, #0x0]\n"
       "80:"  // Height 2: Partial direct writeback: Done
       "b 82f\n"
       "81:"  // Height 2: Full writeback
-      "str q8, [x26, #0x0]\n"
-      "str q9, [x26, #0x10]\n"
-      "str q10, [x26, #0x20]\n"
-      "str q11, [x26, #0x30]\n"
-      "str q12, [x26, #0x40]\n"
-      "str q13, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q14, [x22, #0x0]\n"
-      "str q15, [x22, #0x10]\n"
-      "str q16, [x22, #0x20]\n"
-      "str q17, [x22, #0x30]\n"
-      "str q18, [x22, #0x40]\n"
-      "str q19, [x22, #0x50]\n"
+      "str q8, [x27, #0x0]\n"
+      "str q9, [x27, #0x10]\n"
+      "str q10, [x27, #0x20]\n"
+      "str q11, [x27, #0x30]\n"
+      "str q12, [x27, #0x40]\n"
+      "str q13, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q14, [x23, #0x0]\n"
+      "str q15, [x23, #0x10]\n"
+      "str q16, [x23, #0x20]\n"
+      "str q17, [x23, #0x30]\n"
+      "str q18, [x23, #0x40]\n"
+      "str q19, [x23, #0x50]\n"
       "82:"  // Height 2: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 43b\n"
       "b 166f\n"
       "83:"  // Height 3
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "84:"  // Height 3: Column loop
-      "cbz x27, 85f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 85f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q9, [x27, #0x10]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q10, [x27, #0x20]\n"
-      "ldr q11, [x27, #0x30]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q12, [x27, #0x40]\n"
-      "mov v21.16b, v9.16b\n"
-      "ldr q13, [x27, #0x50]\n"
-      "add x27, x27, #0x60\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "mov v16.16b, v10.16b\n"
       "mov v17.16b, v11.16b\n"
-      "mov v22.16b, v10.16b\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "mov v18.16b, v12.16b\n"
       "mov v19.16b, v13.16b\n"
+      "mov v20.16b, v8.16b\n"
+      "mov v21.16b, v9.16b\n"
+      "add x10, x10, #0x60\n"
+      "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
       "mov v24.16b, v12.16b\n"
       "mov v25.16b, v13.16b\n"
       "b 100f\n"
       "85:"  // Height 3: no bias
       "tbz %x[flags], #0, 99f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 98f\n"
       "tbz x9, #4, 89f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
       "tbz x9, #2, 87f\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x9, #1, 86f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 97f\n"
       "86:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 97f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 97f\n"
       "87:"  // Height 3: Partial accumulate: partial_2_16
       "tbz x9, #1, 88f\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d24, [x21], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d24, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 97f\n"
       "88:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 97f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "b 97f\n"
       "89:"  // Height 3: Partial accumulate: partial_8_0
       "tbz x9, #3, 93f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
       "tbz x9, #2, 91f\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
       "tbz x9, #1, 90f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d23, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 97f\n"
       "90:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 97f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 97f\n"
       "91:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x9, #1, 92f\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d22, [x21], #0x8\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d22, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 97f\n"
       "92:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 97f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 97f\n"
       "93:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x9, #2, 95f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
       "tbz x9, #1, 94f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d21, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 97f\n"
       "94:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 97f\n"
-      "ldr s9, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 97f\n"
       "95:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x9, #1, 96f\n"
-      "ldr d8, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d20, [x21], #0x8\n"
+      "ldr d8, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d20, [x22], #0x8\n"
       "tbz x9, #0, 97f\n"
-      "ld1 { v8.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "ld1 { v8.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 97f\n"
       "96:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s8, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s20, [x22, #0x0]\n"
       "97:"  // Height 3: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 100f\n"
       "98:"  // Height 3: full accumulate
-      "ldr q8, [x26, #0x0]\n"
-      "ldr q9, [x26, #0x10]\n"
-      "ldr q10, [x26, #0x20]\n"
-      "ldr q11, [x26, #0x30]\n"
-      "ldr q12, [x26, #0x40]\n"
-      "ldr q13, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x21, #0x40]\n"
-      "ldr q25, [x21, #0x50]\n"
+      "ldr q8, [x27, #0x0]\n"
+      "ldr q9, [x27, #0x10]\n"
+      "ldr q10, [x27, #0x20]\n"
+      "ldr q11, [x27, #0x30]\n"
+      "ldr q12, [x27, #0x40]\n"
+      "ldr q13, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x22, #0x40]\n"
+      "ldr q25, [x22, #0x50]\n"
       "b 100f\n"
       "99:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1233,64 +1233,64 @@ void a64_hybrid_fp32_mla_4x24 (
       "movi v24.16b, #0x0\n"
       "movi v25.16b, #0x0\n"
       "100:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "101:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 102f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 103f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 103f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 103f\n"
       "102:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "103:"  // Height 3: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 106f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x8\n"
-      "ldr q2, [x21, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x8\n"
+      "ldr q2, [x22, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 105f\n"
       "104:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "sub x25, x25, #0x4\n"
+      "add x24, x24, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
-      "sub x24, x24, #0x4\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "add x23, x23, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x24, #0x8\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "add x22, x22, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "cmp x25, #0x8\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v12.4s, v4.4s, v0.s[0]\n"
       "fmla v18.4s, v4.4s, v1.s[0]\n"
       "fmla v24.4s, v4.4s, v2.s[0]\n"
@@ -1363,40 +1363,40 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v11.4s, v5.4s, v0.s[3]\n"
       "fmla v17.4s, v5.4s, v1.s[3]\n"
       "fmla v23.4s, v5.4s, v2.s[3]\n"
+      "ldr q5, [x28, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
       "fmla v24.4s, v6.4s, v2.s[3]\n"
+      "ldr q6, [x28, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x22, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x21, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q7, [x28, #0x30]\n"
       "bge 104b\n"
       "105:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
+      "add x22, x22, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "sub x25, x25, #0x4\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
@@ -1478,27 +1478,27 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v19.4s, v7.4s, v1.s[3]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
       "106:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 108f\n"
+      "cbz x25, 108f\n"
       "107:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "sub x24, x24, #0x1\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "sub x25, x25, #0x1\n"
+      "ldr s2, [x22], #0x4\n"
       "ldr q4, [x28, #0x0]\n"
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
+      "ldr q5, [x28, #0x10]\n"
       "ldr q6, [x28, #0x20]\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
+      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "ldr q7, [x28, #0x30]\n"
       "ldr q4, [x28, #0x40]\n"
-      "fmla v9.4s, v5.4s, v0.s[0]\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
-      "add x28, x28, #0x60\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
+      "add x28, x28, #0x60\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
@@ -1509,243 +1509,243 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v13.4s, v5.4s, v0.s[0]\n"
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
-      "cbnz x24, 107b\n"
+      "cbnz x25, 107b\n"
       "108:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 101b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 109f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
       "109:"  // Height 3: No activation
       "cmp x9, #0x18\n"
       "bge 122f\n"
       "tbz x9, #4, 113f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v11.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v11.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
       "tbz x9, #2, 111f\n"
-      "st1 { v12.4s }, [x26], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v12.4s }, [x27], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x9, #1, 110f\n"
-      "str d13, [x26], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d13, [x27], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v13.s }[2], [x26]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v13.s }[2], [x27]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 121f\n"
       "110:"  // Height 3: Partial direct writeback: partial_1_20
       "tbz x9, #0, 121f\n"
-      "str s13, [x26, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s13, [x27, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 121f\n"
       "111:"  // Height 3: Partial direct writeback: partial_2_16
       "tbz x9, #1, 112f\n"
-      "str d12, [x26], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d12, [x27], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v12.s }[2], [x26]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v12.s }[2], [x27]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 121f\n"
       "112:"  // Height 3: Partial direct writeback: partial_1_16
       "tbz x9, #0, 121f\n"
-      "str s12, [x26, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s12, [x27, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "b 121f\n"
       "113:"  // Height 3: Partial direct writeback: partial_8_0
       "tbz x9, #3, 117f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
       "tbz x9, #2, 115f\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
       "tbz x9, #1, 114f\n"
-      "str d11, [x26], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
+      "str d11, [x27], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v11.s }[2], [x26]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "st1 { v11.s }[2], [x27]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 121f\n"
       "114:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 121f\n"
-      "str s11, [x26, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "str s11, [x27, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 121f\n"
       "115:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 116f\n"
-      "str d10, [x26], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
+      "str d10, [x27], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v10.s }[2], [x26]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "st1 { v10.s }[2], [x27]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 121f\n"
       "116:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 121f\n"
-      "str s10, [x26, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "str s10, [x27, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 121f\n"
       "117:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 119f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
       "tbz x9, #1, 118f\n"
-      "str d9, [x26], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
+      "str d9, [x27], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v9.s }[2], [x26]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "st1 { v9.s }[2], [x27]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 121f\n"
       "118:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 121f\n"
-      "str s9, [x26, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "str s9, [x27, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 121f\n"
       "119:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 120f\n"
-      "str d8, [x26], #0x8\n"
-      "str d14, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
+      "str d8, [x27], #0x8\n"
+      "str d14, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
       "tbz x9, #0, 121f\n"
-      "st1 { v8.s }[2], [x26]\n"
-      "st1 { v14.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "st1 { v8.s }[2], [x27]\n"
+      "st1 { v14.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 121f\n"
       "120:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x26, #0x0]\n"
-      "str s14, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s8, [x27, #0x0]\n"
+      "str s14, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "121:"  // Height 3: Partial direct writeback: Done
       "b 123f\n"
       "122:"  // Height 3: Full writeback
-      "str q8, [x26, #0x0]\n"
-      "str q9, [x26, #0x10]\n"
-      "str q10, [x26, #0x20]\n"
-      "str q11, [x26, #0x30]\n"
-      "str q12, [x26, #0x40]\n"
-      "str q13, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q14, [x22, #0x0]\n"
-      "str q15, [x22, #0x10]\n"
-      "str q16, [x22, #0x20]\n"
-      "str q17, [x22, #0x30]\n"
-      "str q18, [x22, #0x40]\n"
-      "str q19, [x22, #0x50]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x21, #0x40]\n"
-      "str q25, [x21, #0x50]\n"
+      "str q8, [x27, #0x0]\n"
+      "str q9, [x27, #0x10]\n"
+      "str q10, [x27, #0x20]\n"
+      "str q11, [x27, #0x30]\n"
+      "str q12, [x27, #0x40]\n"
+      "str q13, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q14, [x23, #0x0]\n"
+      "str q15, [x23, #0x10]\n"
+      "str q16, [x23, #0x20]\n"
+      "str q17, [x23, #0x30]\n"
+      "str q18, [x23, #0x40]\n"
+      "str q19, [x23, #0x50]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x22, #0x40]\n"
+      "str q25, [x22, #0x50]\n"
       "123:"  // Height 3: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 84b\n"
       "b 166f\n"
       "124:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x10\n"
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x10\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "125:"  // Height 4: Column loop
-      "cbz x27, 126f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 126f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "mov v14.16b, v8.16b\n"
-      "ldr q9, [x27, #0x10]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q10, [x27, #0x20]\n"
-      "mov v26.16b, v8.16b\n"
-      "ldr q11, [x27, #0x30]\n"
-      "ldr q12, [x27, #0x40]\n"
       "mov v15.16b, v9.16b\n"
-      "ldr q13, [x27, #0x50]\n"
-      "add x27, x27, #0x60\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "mov v16.16b, v10.16b\n"
-      "mov v21.16b, v9.16b\n"
       "mov v17.16b, v11.16b\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "mov v18.16b, v12.16b\n"
       "mov v19.16b, v13.16b\n"
+      "mov v20.16b, v8.16b\n"
+      "mov v21.16b, v9.16b\n"
+      "add x10, x10, #0x60\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
       "mov v24.16b, v12.16b\n"
       "mov v25.16b, v13.16b\n"
+      "mov v26.16b, v8.16b\n"
       "mov v27.16b, v9.16b\n"
       "mov v28.16b, v10.16b\n"
       "mov v29.16b, v11.16b\n"
@@ -1754,204 +1754,204 @@ void a64_hybrid_fp32_mla_4x24 (
       "b 141f\n"
       "126:"  // Height 4: no bias
       "tbz %x[flags], #0, 140f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 139f\n"
       "tbz x9, #4, 130f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x9, #2, 128f\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x9, #1, 127f\n"
-      "mov x19, #0x58\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 138f\n"
       "127:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 138f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 138f\n"
       "128:"  // Height 4: Partial accumulate: partial_2_16
       "tbz x9, #1, 129f\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 138f\n"
       "129:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 138f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 138f\n"
       "130:"  // Height 4: Partial accumulate: partial_8_0
       "tbz x9, #3, 134f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
       "tbz x9, #2, 132f\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x9, #1, 131f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 138f\n"
       "131:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 138f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 138f\n"
       "132:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x9, #1, 133f\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 138f\n"
       "133:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 138f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "b 138f\n"
       "134:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x9, #2, 136f\n"
-      "ld1 { v8.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
       "tbz x9, #1, 135f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 138f\n"
       "135:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 138f\n"
-      "ldr s9, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 138f\n"
       "136:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x9, #1, 137f\n"
-      "ldr d8, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
+      "ldr d8, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
       "tbz x9, #0, 138f\n"
-      "ld1 { v8.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "ld1 { v8.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 138f\n"
       "137:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "ldr s8, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "138:"  // Height 4: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 141f\n"
       "139:"  // Height 4: full accumulate
-      "ldr q8, [x26, #0x0]\n"
-      "ldr q9, [x26, #0x10]\n"
-      "ldr q10, [x26, #0x20]\n"
-      "ldr q11, [x26, #0x30]\n"
-      "ldr q12, [x26, #0x40]\n"
-      "ldr q13, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x21, #0x40]\n"
-      "ldr q25, [x21, #0x50]\n"
-      "ldr q26, [x20, #0x0]\n"
-      "ldr q27, [x20, #0x10]\n"
-      "ldr q28, [x20, #0x20]\n"
-      "ldr q29, [x20, #0x30]\n"
-      "ldr q30, [x20, #0x40]\n"
-      "ldr q31, [x20, #0x50]\n"
+      "ldr q8, [x27, #0x0]\n"
+      "ldr q9, [x27, #0x10]\n"
+      "ldr q10, [x27, #0x20]\n"
+      "ldr q11, [x27, #0x30]\n"
+      "ldr q12, [x27, #0x40]\n"
+      "ldr q13, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x22, #0x40]\n"
+      "ldr q25, [x22, #0x50]\n"
+      "ldr q26, [x21, #0x0]\n"
+      "ldr q27, [x21, #0x10]\n"
+      "ldr q28, [x21, #0x20]\n"
+      "ldr q29, [x21, #0x30]\n"
+      "ldr q30, [x21, #0x40]\n"
+      "ldr q31, [x21, #0x50]\n"
       "b 141f\n"
       "140:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1979,71 +1979,71 @@ void a64_hybrid_fp32_mla_4x24 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "141:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "142:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 143f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 144f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 144f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 144f\n"
       "143:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "144:"  // Height 4: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 147f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x8\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x8\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 146f\n"
       "145:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "sub x25, x25, #0x4\n"
+      "add x24, x24, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
-      "add x20, x20, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x4\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x8\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v27.4s, v5.4s, v3.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "cmp x25, #0x8\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "fmla v28.4s, v6.4s, v3.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
       "fmla v29.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x28, #0x70]\n"
@@ -2137,47 +2137,47 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v17.4s, v5.4s, v1.s[3]\n"
       "fmla v23.4s, v5.4s, v2.s[3]\n"
       "fmla v29.4s, v5.4s, v3.s[3]\n"
+      "ldr q5, [x28, #0x10]\n"
       "fmla v12.4s, v6.4s, v0.s[3]\n"
       "fmla v18.4s, v6.4s, v1.s[3]\n"
       "fmla v24.4s, v6.4s, v2.s[3]\n"
       "fmla v30.4s, v6.4s, v3.s[3]\n"
+      "ldr q6, [x28, #0x20]\n"
       "fmla v13.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "fmla v19.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x22, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
       "fmla v25.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x21, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
       "fmla v31.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x20, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q7, [x28, #0x30]\n"
       "bge 145b\n"
       "146:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.4s, v4.4s, v0.s[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v20.4s, v4.4s, v2.s[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x20, x20, #0x10\n"
       "fmla v15.4s, v5.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x25, x25, #0x4\n"
       "fmla v21.4s, v5.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v27.4s, v5.4s, v3.s[0]\n"
       "ldr q5, [x28, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v16.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v22.4s, v6.4s, v2.s[0]\n"
       "fmla v28.4s, v6.4s, v3.s[0]\n"
       "ldr q6, [x28, #0x60]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v17.4s, v7.4s, v1.s[0]\n"
       "fmla v23.4s, v7.4s, v2.s[0]\n"
@@ -2281,20 +2281,20 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v25.4s, v7.4s, v2.s[3]\n"
       "fmla v31.4s, v7.4s, v3.s[3]\n"
       "147:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 149f\n"
+      "cbz x25, 149f\n"
       "148:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "sub x24, x24, #0x1\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "sub x25, x25, #0x1\n"
+      "ldr s2, [x22], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "ldr q4, [x28, #0x0]\n"
-      "fmla v8.4s, v4.4s, v0.s[0]\n"
       "ldr q5, [x28, #0x10]\n"
+      "fmla v8.4s, v4.4s, v0.s[0]\n"
       "fmla v14.4s, v4.4s, v1.s[0]\n"
       "ldr q6, [x28, #0x20]\n"
-      "fmla v20.4s, v4.4s, v2.s[0]\n"
       "ldr q7, [x28, #0x30]\n"
+      "fmla v20.4s, v4.4s, v2.s[0]\n"
       "fmla v26.4s, v4.4s, v3.s[0]\n"
       "ldr q4, [x28, #0x40]\n"
       "fmla v9.4s, v5.4s, v0.s[0]\n"
@@ -2319,275 +2319,275 @@ void a64_hybrid_fp32_mla_4x24 (
       "fmla v19.4s, v5.4s, v1.s[0]\n"
       "fmla v25.4s, v5.4s, v2.s[0]\n"
       "fmla v31.4s, v5.4s, v3.s[0]\n"
-      "cbnz x24, 148b\n"
+      "cbnz x25, 148b\n"
       "149:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 142b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbz %x[flags], #1, 150f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v31.4s, v31.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v31.4s, v31.4s, v0.4s\n"
       "150:"  // Height 4: No activation
       "cmp x9, #0x18\n"
       "bge 163f\n"
       "tbz x9, #4, 154f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v11.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v27.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v11.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v27.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
       "tbz x9, #2, 152f\n"
-      "st1 { v12.4s }, [x26], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v30.4s }, [x20], #0x10\n"
+      "st1 { v12.4s }, [x27], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v30.4s }, [x21], #0x10\n"
       "tbz x9, #1, 151f\n"
-      "str d13, [x26], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
+      "str d13, [x27], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v13.s }[2], [x26]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
-      "st1 { v31.s }[2], [x20]\n"
+      "st1 { v13.s }[2], [x27]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
+      "st1 { v31.s }[2], [x21]\n"
       "b 162f\n"
       "151:"  // Height 4: Partial direct writeback: partial_1_20
       "tbz x9, #0, 162f\n"
-      "str s13, [x26, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
-      "str s31, [x20, #0x0]\n"
+      "str s13, [x27, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
+      "str s31, [x21, #0x0]\n"
       "b 162f\n"
       "152:"  // Height 4: Partial direct writeback: partial_2_16
       "tbz x9, #1, 153f\n"
-      "str d12, [x26], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
+      "str d12, [x27], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v12.s }[2], [x26]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
+      "st1 { v12.s }[2], [x27]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
       "b 162f\n"
       "153:"  // Height 4: Partial direct writeback: partial_1_16
       "tbz x9, #0, 162f\n"
-      "str s12, [x26, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
+      "str s12, [x27, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
       "b 162f\n"
       "154:"  // Height 4: Partial direct writeback: partial_8_0
       "tbz x9, #3, 158f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v27.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v27.4s }, [x21], #0x10\n"
       "tbz x9, #2, 156f\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
       "tbz x9, #1, 155f\n"
-      "str d11, [x26], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
+      "str d11, [x27], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v11.s }[2], [x26]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
+      "st1 { v11.s }[2], [x27]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
       "b 162f\n"
       "155:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 162f\n"
-      "str s11, [x26, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
+      "str s11, [x27, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
       "b 162f\n"
       "156:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 157f\n"
-      "str d10, [x26], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
+      "str d10, [x27], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v10.s }[2], [x26]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
+      "st1 { v10.s }[2], [x27]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
       "b 162f\n"
       "157:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 162f\n"
-      "str s10, [x26, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
+      "str s10, [x27, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
       "b 162f\n"
       "158:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 160f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
       "tbz x9, #1, 159f\n"
-      "str d9, [x26], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
+      "str d9, [x27], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v9.s }[2], [x26]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "st1 { v9.s }[2], [x27]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 162f\n"
       "159:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 162f\n"
-      "str s9, [x26, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "str s9, [x27, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 162f\n"
       "160:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 161f\n"
-      "str d8, [x26], #0x8\n"
-      "str d14, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
+      "str d8, [x27], #0x8\n"
+      "str d14, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
       "tbz x9, #0, 162f\n"
-      "st1 { v8.s }[2], [x26]\n"
-      "st1 { v14.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "st1 { v8.s }[2], [x27]\n"
+      "st1 { v14.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 162f\n"
       "161:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x26, #0x0]\n"
-      "str s14, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "str s8, [x27, #0x0]\n"
+      "str s14, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "162:"  // Height 4: Partial direct writeback: Done
       "b 164f\n"
       "163:"  // Height 4: Full writeback
-      "str q8, [x26, #0x0]\n"
-      "str q9, [x26, #0x10]\n"
-      "str q10, [x26, #0x20]\n"
-      "str q11, [x26, #0x30]\n"
-      "str q12, [x26, #0x40]\n"
-      "str q13, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q14, [x22, #0x0]\n"
-      "str q15, [x22, #0x10]\n"
-      "str q16, [x22, #0x20]\n"
-      "str q17, [x22, #0x30]\n"
-      "str q18, [x22, #0x40]\n"
-      "str q19, [x22, #0x50]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x21, #0x40]\n"
-      "str q25, [x21, #0x50]\n"
-      "str q26, [x20, #0x0]\n"
-      "str q27, [x20, #0x10]\n"
-      "str q28, [x20, #0x20]\n"
-      "str q29, [x20, #0x30]\n"
-      "str q30, [x20, #0x40]\n"
-      "str q31, [x20, #0x50]\n"
+      "str q8, [x27, #0x0]\n"
+      "str q9, [x27, #0x10]\n"
+      "str q10, [x27, #0x20]\n"
+      "str q11, [x27, #0x30]\n"
+      "str q12, [x27, #0x40]\n"
+      "str q13, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q14, [x23, #0x0]\n"
+      "str q15, [x23, #0x10]\n"
+      "str q16, [x23, #0x20]\n"
+      "str q17, [x23, #0x30]\n"
+      "str q18, [x23, #0x40]\n"
+      "str q19, [x23, #0x50]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x22, #0x40]\n"
+      "str q25, [x22, #0x50]\n"
+      "str q26, [x21, #0x0]\n"
+      "str q27, [x21, #0x10]\n"
+      "str q28, [x21, #0x20]\n"
+      "str q29, [x21, #0x30]\n"
+      "str q30, [x21, #0x40]\n"
+      "str q31, [x21, #0x50]\n"
       "164:"  // Height 4: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 125b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 166f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 165f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "165:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "166:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp
index e8b7db21bd..985d57d9b6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -102,82 +102,82 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 67f\n"
       "beq 34f\n"
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x16, %x[bias]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x16, 3f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
-      "ldr q11, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
+      "cbz x7, 3f\n"
+      "ldr q8, [x7, #0x0]\n"
+      "ldr q9, [x7, #0x10]\n"
+      "ldr q10, [x7, #0x20]\n"
+      "ldr q11, [x7, #0x30]\n"
+      "add x7, x7, #0x40\n"
       "b 14f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 13f\n"
       "cmp x8, #0x10\n"
       "bge 12f\n"
       "tbz x8, #3, 7f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
       "tbz x8, #2, 5f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "tbz x8, #1, 4f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x8, #0, 11f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "b 11f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 11f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "b 11f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x8, #1, 6f\n"
-      "ldr d10, [x15], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x8, #0, 11f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "b 11f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 11f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "b 11f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x8, #2, 9f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "tbz x8, #1, 8f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x8, #0, 11f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "b 11f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 11f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "b 11f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x8, #1, 10f\n"
-      "ldr d8, [x15], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x8, #0, 11f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "b 11f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "11:"  // Height 1: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 14f\n"
       "12:"  // Height 1: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "b 14f\n"
       "13:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -185,112 +185,109 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "14:"  // Height 1: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "15:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "cbnz x14, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "cbnz x15, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
       "b 17f\n"
       "16:"  // Height 1: setup direct input
-      "mov x12, %x[input_ptr]\n"
+      "mov x13, %x[input_ptr]\n"
       "17:"  // Height 1: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 20f\n"
-      "ldr q0, [x12, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
       "ldr q6, [x17, #0x0]\n"
-      "cmp x13, #0x8\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 19f\n"
       "18:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
-      "ldr x11, [x17, #0x18]\n"
-      "add x12, x12, #0x10\n"
       "ldr d6, [x17, #0x20]\n"
-      "sub x13, x13, #0x4\n"
-      "ldr x10, [x17, #0x28]\n"
-      "cmp x13, #0x8\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x38]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr d6, [x17, #0x40]\n"
-      "ldr x10, [x17, #0x48]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x58]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr d6, [x17, #0x60]\n"
-      "ldr x10, [x17, #0x68]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x78]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "ldr d6, [x17, #0x80]\n"
-      "ldr x10, [x17, #0x88]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x98]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x98]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "ldr d6, [x17, #0xa0]\n"
-      "ldr x10, [x17, #0xa8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xb8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "ldr d6, [x17, #0xc0]\n"
-      "ldr x10, [x17, #0xc8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "ldr d6, [x17, #0xe0]\n"
-      "ldr x10, [x17, #0xe8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xf8]\n"
+      "mov v7.d[1], x11\n"
+      "add x13, x13, #0x10\n"
       "add x17, x17, #0x100\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "ldr d6, [x17, #0x0]\n"
-      "ldr x10, [x17, #0x8]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d0, [x12, #0x0]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d0, [x13, #0x0]\n"
+      "sub x14, x14, #0x4\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x8\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "bge 18b\n"
       "19:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "ldr q6, [x17, #0x20]\n"
-      "sub x13, x13, #0x4\n"
-      "add x12, x12, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x17, #0x50]\n"
@@ -314,201 +311,204 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "ldr q6, [x17, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x17, #0xf0]\n"
+      "add x13, x13, #0x10\n"
+      "sub x14, x14, #0x4\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
-      "add x17, x17, #0x100\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
+      "add x17, x17, #0x100\n"
       "20:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x13, 22f\n"
+      "cbz x14, 22f\n"
       "21:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q6, [x17, #0x20]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x30]\n"
-      "add x17, x17, #0x40\n"
+      "ldr q6, [x17, #0x20]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "cbnz x13, 21b\n"
+      "add x17, x17, #0x40\n"
+      "cbnz x14, 21b\n"
       "22:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 15b\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "tbz %x[flags], #1, 23f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
       "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
       "23:"  // Height 1: No activation
       "cmp x8, #0x10\n"
       "bge 32f\n"
       "tbz x8, #3, 27f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "tbz x8, #2, 25f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "tbz x8, #1, 24f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "tbz x8, #0, 31f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "b 31f\n"
       "24:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x8, #0, 31f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "b 31f\n"
       "25:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x8, #1, 26f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "tbz x8, #0, 31f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "b 31f\n"
       "26:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x8, #0, 31f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "b 31f\n"
       "27:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x8, #2, 29f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "tbz x8, #1, 28f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "tbz x8, #0, 31f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "b 31f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x8, #0, 31f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "b 31f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x8, #1, 30f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "tbz x8, #0, 31f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "b 31f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "31:"  // Height 1: Partial direct writeback: Done
       "b 33f\n"
       "32:"  // Height 1: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "33:"  // Height 1: Writeback done
       "subs x8, x8, #0x10\n"
       "bgt 2b\n"
       "b 200f\n"
       "34:"  // Height 2
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "35:"  // Height 2: Column loop
-      "cbz x16, 36f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 36f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
+      "add x7, x7, #0x40\n"
       "b 47f\n"
       "36:"  // Height 2: no bias
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x8, #0x10\n"
-      "add x25, x15, x19, LSL #2\n"
+      "add x25, x16, x20, LSL #2\n"
       "bge 45f\n"
       "tbz x8, #3, 40f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
       "ld1 { v13.4s }, [x25], #0x10\n"
       "tbz x8, #2, 38f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "ld1 { v14.4s }, [x25], #0x10\n"
       "tbz x8, #1, 37f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d15, [x25], #0x8\n"
       "tbz x8, #0, 44f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "ld1 { v15.s }[2], [x25]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 44f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "ldr s15, [x25, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x8, #1, 39f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d14, [x25], #0x8\n"
-      "mov x19, #0x28\n"
       "tbz x8, #0, 44f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "ld1 { v14.s }[2], [x25]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 44f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "ldr s14, [x25, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x8, #2, 42f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "tbz x8, #1, 41f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d13, [x25], #0x8\n"
       "tbz x8, #0, 44f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "ld1 { v13.s }[2], [x25]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 44f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "ldr s13, [x25, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x8, #1, 43f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d12, [x25], #0x8\n"
-      "mov x19, #0x8\n"
       "tbz x8, #0, 44f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "ld1 { v12.s }[2], [x25]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s12, [x25, #0x0]\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -524,147 +524,147 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "cbnz x14, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
-      "add x28, x28, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "cbnz x15, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
+      "add x9, x9, x20, LSL #2\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #2\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #2\n"
       "50:"  // Height 2: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 53f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x8\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
+      "ldr q1, [x9, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x18]\n"
       "ldr d6, [x17, #0x20]\n"
-      "add x12, x12, #0x10\n"
-      "ldr x10, [x17, #0x28]\n"
-      "add x28, x28, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "sub x13, x13, #0x4\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "ldr d6, [x17, #0x40]\n"
-      "ldr x10, [x17, #0x48]\n"
-      "cmp x13, #0x8\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x58]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "ldr d6, [x17, #0x60]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x78]\n"
-      "ldr x27, [x28, #0x8]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "ldr d6, [x17, #0x80]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x98]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x98]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
       "ldr d6, [x17, #0xa0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
       "ldr d6, [x17, #0xc0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
       "ldr d6, [x17, #0xe0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "ldr d7, [x17, #0xf0]\n"
-      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "mov v6.d[1], x12\n"
+      "add x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
       "add x17, x17, #0x100\n"
+      "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "ldr d6, [x17, #0x0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d1, [x28, #0x0]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
+      "ldr d1, [x9, #0x0]\n"
+      "sub x14, x14, #0x4\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x8\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x28, [x9, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
-      "sub x13, x13, #0x4\n"
-      "add x12, x12, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x9, x9, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x4\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x17, #0x50]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "ldr q6, [x17, #0x60]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
@@ -700,14 +700,14 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v11.4s, v7.4s, v0.s[3]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x13, 55f\n"
+      "cbz x14, 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
+      "ldr s1, [x9], #0x4\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
@@ -718,21 +718,19 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "cbnz x13, 54b\n"
+      "cbnz x14, 54b\n"
       "55:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 56f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -741,85 +739,87 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v13.4s, v13.4s, v0.4s\n"
       "fmin v14.4s, v14.4s, v0.4s\n"
       "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
       "56:"  // Height 2: No activation
       "cmp x8, #0x10\n"
       "bge 65f\n"
       "tbz x8, #3, 60f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v13.4s }, [x25], #0x10\n"
       "tbz x8, #2, 58f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "st1 { v14.4s }, [x25], #0x10\n"
       "tbz x8, #1, 57f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "tbz x8, #0, 64f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "st1 { v15.s }[2], [x25]\n"
       "b 64f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x8, #0, 64f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "str s15, [x25, #0x0]\n"
       "b 64f\n"
       "58:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x8, #1, 59f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "tbz x8, #0, 64f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "st1 { v14.s }[2], [x25]\n"
       "b 64f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x8, #0, 64f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "str s14, [x25, #0x0]\n"
       "b 64f\n"
       "60:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x8, #2, 62f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "tbz x8, #1, 61f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "tbz x8, #0, 64f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "st1 { v13.s }[2], [x25]\n"
       "b 64f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x8, #0, 64f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "str s13, [x25, #0x0]\n"
       "b 64f\n"
       "62:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x8, #1, 63f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "tbz x8, #0, 64f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "st1 { v12.s }[2], [x25]\n"
       "b 64f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "str s12, [x25, #0x0]\n"
       "64:"  // Height 2: Partial direct writeback: Done
       "b 66f\n"
       "65:"  // Height 2: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -829,125 +829,125 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "bgt 35b\n"
       "b 200f\n"
       "67:"  // Height 3
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "68:"  // Height 3: Column loop
-      "cbz x16, 69f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 69f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 80f\n"
       "69:"  // Height 3: no bias
       "tbz %x[flags], #0, 79f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
       "cmp x8, #0x10\n"
-      "add x25, x15, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 78f\n"
       "tbz x8, #3, 73f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
       "ld1 { v13.4s }, [x25], #0x10\n"
       "ld1 { v17.4s }, [x24], #0x10\n"
       "tbz x8, #2, 71f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "ld1 { v14.4s }, [x25], #0x10\n"
       "ld1 { v18.4s }, [x24], #0x10\n"
       "tbz x8, #1, 70f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "tbz x8, #0, 77f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "ld1 { v15.s }[2], [x25]\n"
       "ld1 { v19.s }[2], [x24]\n"
       "b 77f\n"
       "70:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 77f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "ldr s15, [x25, #0x0]\n"
       "ldr s19, [x24, #0x0]\n"
       "b 77f\n"
       "71:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x8, #1, 72f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d14, [x25], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d18, [x24], #0x8\n"
       "tbz x8, #0, 77f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "ld1 { v14.s }[2], [x25]\n"
       "ld1 { v18.s }[2], [x24]\n"
       "b 77f\n"
       "72:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 77f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "ldr s14, [x25, #0x0]\n"
       "ldr s18, [x24, #0x0]\n"
       "b 77f\n"
       "73:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x8, #2, 75f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
       "tbz x8, #1, 74f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "tbz x8, #0, 77f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "ld1 { v13.s }[2], [x25]\n"
       "ld1 { v17.s }[2], [x24]\n"
       "b 77f\n"
       "74:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 77f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "ldr s13, [x25, #0x0]\n"
       "ldr s17, [x24, #0x0]\n"
       "b 77f\n"
       "75:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x8, #1, 76f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d12, [x25], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "tbz x8, #0, 77f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "ld1 { v12.s }[2], [x25]\n"
       "ld1 { v16.s }[2], [x24]\n"
       "b 77f\n"
       "76:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s12, [x25, #0x0]\n"
       "ldr s16, [x24, #0x0]\n"
       "77:"  // Height 3: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 80f\n"
       "78:"  // Height 3: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -971,175 +971,175 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "80:"  // Height 3: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "81:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 82f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "cbnz x14, 83f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
-      "add x28, x28, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "cbnz x15, 83f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
+      "add x9, x9, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
       "b 83f\n"
       "82:"  // Height 3: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #2\n"
-      "add x26, x28, x19, LSL #2\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #2\n"
+      "add x27, x9, x20, LSL #2\n"
       "83:"  // Height 3: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 86f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x8\n"
-      "ldr q2, [x26, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 85f\n"
       "84:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "ldr d6, [x17, #0x20]\n"
-      "ldr x10, [x17, #0x28]\n"
-      "add x12, x12, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x17, #0x38]\n"
-      "add x28, x28, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr x10, [x17, #0x48]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x40]\n"
-      "add x26, x26, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x50]\n"
-      "sub x13, x13, #0x4\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x68]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x60]\n"
-      "cmp x13, #0x8\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x88]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
+      "add x13, x13, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0xf0]\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
+      "add x27, x27, #0x10\n"
       "add x17, x17, #0x100\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
-      "mov v0.d[1], x9\n"
-      "ldr d2, [x26, #0x0]\n"
-      "mov v1.d[1], x27\n"
-      "mov v2.d[1], x25\n"
+      "ldr d2, [x27, #0x0]\n"
+      "sub x14, x14, #0x4\n"
+      "ldr d7, [x17, #0x10]\n"
+      "cmp x14, #0x8\n"
+      "ldr x26, [x27, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x17, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
+      "mov v2.d[1], x26\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "mov v7.d[1], x11\n"
       "bge 84b\n"
       "85:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x13, x13, #0x4\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "sub x14, x14, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x17, #0x50]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
@@ -1190,15 +1190,15 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "86:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x13, 88f\n"
+      "cbz x14, 88f\n"
       "87:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
-      "ldr s1, [x28], #0x4\n"
-      "ldr s2, [x26], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
+      "ldr s1, [x9], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
@@ -1213,23 +1213,21 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "cbnz x13, 87b\n"
+      "cbnz x14, 87b\n"
       "88:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 81b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
       "tbz %x[flags], #1, 89f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -1240,107 +1238,109 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "89:"  // Height 3: No activation
       "cmp x8, #0x10\n"
       "bge 98f\n"
       "tbz x8, #3, 93f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v13.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
       "st1 { v17.4s }, [x24], #0x10\n"
       "tbz x8, #2, 91f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "st1 { v14.4s }, [x25], #0x10\n"
       "st1 { v18.4s }, [x24], #0x10\n"
       "tbz x8, #1, 90f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "tbz x8, #0, 97f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "st1 { v15.s }[2], [x25]\n"
       "st1 { v19.s }[2], [x24]\n"
       "b 97f\n"
       "90:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x8, #0, 97f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "str s15, [x25, #0x0]\n"
       "str s19, [x24, #0x0]\n"
       "b 97f\n"
       "91:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x8, #1, 92f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "tbz x8, #0, 97f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "st1 { v14.s }[2], [x25]\n"
       "st1 { v18.s }[2], [x24]\n"
       "b 97f\n"
       "92:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x8, #0, 97f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "str s14, [x25, #0x0]\n"
       "str s18, [x24, #0x0]\n"
       "b 97f\n"
       "93:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x8, #2, 95f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
       "tbz x8, #1, 94f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "tbz x8, #0, 97f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "st1 { v13.s }[2], [x25]\n"
       "st1 { v17.s }[2], [x24]\n"
       "b 97f\n"
       "94:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x8, #0, 97f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "str s13, [x25, #0x0]\n"
       "str s17, [x24, #0x0]\n"
       "b 97f\n"
       "95:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x8, #1, 96f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "tbz x8, #0, 97f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "st1 { v12.s }[2], [x25]\n"
       "st1 { v16.s }[2], [x24]\n"
       "b 97f\n"
       "96:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "str s12, [x25, #0x0]\n"
       "str s16, [x24, #0x0]\n"
       "97:"  // Height 3: Partial direct writeback: Done
       "b 99f\n"
       "98:"  // Height 3: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -1354,22 +1354,22 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "bgt 68b\n"
       "b 200f\n"
       "100:"  // Height 4
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "101:"  // Height 4: Column loop
-      "cbz x16, 102f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 102f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -1380,120 +1380,120 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 113f\n"
       "102:"  // Height 4: no bias
       "tbz %x[flags], #0, 112f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x8, #0x10\n"
-      "add x25, x15, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 111f\n"
       "tbz x8, #3, 106f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
       "ld1 { v13.4s }, [x25], #0x10\n"
       "ld1 { v17.4s }, [x24], #0x10\n"
-      "ld1 { v20.4s }, [x23], #0x10\n"
       "ld1 { v21.4s }, [x23], #0x10\n"
       "tbz x8, #2, 104f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "ld1 { v14.4s }, [x25], #0x10\n"
       "ld1 { v18.4s }, [x24], #0x10\n"
       "ld1 { v22.4s }, [x23], #0x10\n"
       "tbz x8, #1, 103f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "tbz x8, #0, 110f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "ld1 { v15.s }[2], [x25]\n"
       "ld1 { v19.s }[2], [x24]\n"
       "ld1 { v23.s }[2], [x23]\n"
       "b 110f\n"
       "103:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 110f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "ldr s15, [x25, #0x0]\n"
       "ldr s19, [x24, #0x0]\n"
       "ldr s23, [x23, #0x0]\n"
       "b 110f\n"
       "104:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x8, #1, 105f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d14, [x25], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "tbz x8, #0, 110f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "ld1 { v14.s }[2], [x25]\n"
       "ld1 { v18.s }[2], [x24]\n"
       "ld1 { v22.s }[2], [x23]\n"
       "b 110f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 110f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "ldr s14, [x25, #0x0]\n"
       "ldr s18, [x24, #0x0]\n"
       "ldr s22, [x23, #0x0]\n"
       "b 110f\n"
       "106:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x8, #2, 108f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
       "ld1 { v20.4s }, [x23], #0x10\n"
       "tbz x8, #1, 107f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "tbz x8, #0, 110f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "ld1 { v13.s }[2], [x25]\n"
       "ld1 { v17.s }[2], [x24]\n"
       "ld1 { v21.s }[2], [x23]\n"
       "b 110f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 110f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "ldr s13, [x25, #0x0]\n"
       "ldr s17, [x24, #0x0]\n"
       "ldr s21, [x23, #0x0]\n"
       "b 110f\n"
       "108:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x8, #1, 109f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d12, [x25], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "tbz x8, #0, 110f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "ld1 { v12.s }[2], [x25]\n"
       "ld1 { v16.s }[2], [x24]\n"
       "ld1 { v20.s }[2], [x23]\n"
       "b 110f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s12, [x25, #0x0]\n"
       "ldr s16, [x24, #0x0]\n"
       "ldr s20, [x23, #0x0]\n"
       "110:"  // Height 4: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 113f\n"
       "111:"  // Height 4: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -1525,204 +1525,204 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "113:"  // Height 4: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "114:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 115f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x14, 116f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
-      "add x28, x28, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x15, 116f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
+      "add x9, x9, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 116f\n"
       "115:"  // Height 4: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #2\n"
-      "add x26, x28, x19, LSL #2\n"
-      "add x24, x26, x19, LSL #2\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #2\n"
+      "add x27, x9, x20, LSL #2\n"
+      "add x25, x27, x20, LSL #2\n"
       "116:"  // Height 4: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 119f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x8\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 118f\n"
       "117:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
       "ldr d6, [x17, #0x20]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x17, #0x38]\n"
-      "add x12, x12, #0x10\n"
-      "add x28, x28, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x9, x9, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "ldr x10, [x17, #0x48]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x68]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "add x26, x26, #0x10\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr d6, [x17, #0x60]\n"
+      "fmla v9.4s, v7.4s, v0.s[1]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x70]\n"
-      "add x24, x24, #0x10\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
-      "sub x13, x13, #0x4\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x4\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
-      "cmp x13, #0x8\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
+      "cmp x14, #0x8\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "ldr d7, [x17, #0xf0]\n"
-      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x100\n"
+      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
+      "ldr d3, [x25, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v7.d[1], x11\n"
       "bge 117b\n"
       "118:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x13, x13, #0x4\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
@@ -1787,16 +1787,16 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "119:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x13, 121f\n"
+      "cbz x14, 121f\n"
       "120:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
-      "ldr s1, [x28], #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
+      "ldr s1, [x9], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
@@ -1815,25 +1815,23 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "cbnz x13, 120b\n"
+      "cbnz x14, 120b\n"
       "121:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 114b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 122f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -1844,34 +1842,36 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
       "fmin v20.4s, v20.4s, v0.4s\n"
       "fmin v21.4s, v21.4s, v0.4s\n"
       "fmin v22.4s, v22.4s, v0.4s\n"
       "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
       "122:"  // Height 4: No activation
       "cmp x8, #0x10\n"
       "bge 131f\n"
       "tbz x8, #3, 126f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v13.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
@@ -1879,96 +1879,96 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "st1 { v20.4s }, [x23], #0x10\n"
       "st1 { v21.4s }, [x23], #0x10\n"
       "tbz x8, #2, 124f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "st1 { v14.4s }, [x25], #0x10\n"
       "st1 { v18.4s }, [x24], #0x10\n"
       "st1 { v22.4s }, [x23], #0x10\n"
       "tbz x8, #1, 123f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "tbz x8, #0, 130f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "st1 { v15.s }[2], [x25]\n"
       "st1 { v19.s }[2], [x24]\n"
       "st1 { v23.s }[2], [x23]\n"
       "b 130f\n"
       "123:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x8, #0, 130f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "str s15, [x25, #0x0]\n"
       "str s19, [x24, #0x0]\n"
       "str s23, [x23, #0x0]\n"
       "b 130f\n"
       "124:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x8, #1, 125f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "tbz x8, #0, 130f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "st1 { v14.s }[2], [x25]\n"
       "st1 { v18.s }[2], [x24]\n"
       "st1 { v22.s }[2], [x23]\n"
       "b 130f\n"
       "125:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x8, #0, 130f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "str s14, [x25, #0x0]\n"
       "str s18, [x24, #0x0]\n"
       "str s22, [x23, #0x0]\n"
       "b 130f\n"
       "126:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x8, #2, 128f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
       "st1 { v20.4s }, [x23], #0x10\n"
       "tbz x8, #1, 127f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "tbz x8, #0, 130f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "st1 { v13.s }[2], [x25]\n"
       "st1 { v17.s }[2], [x24]\n"
       "st1 { v21.s }[2], [x23]\n"
       "b 130f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x8, #0, 130f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "str s13, [x25, #0x0]\n"
       "str s17, [x24, #0x0]\n"
       "str s21, [x23, #0x0]\n"
       "b 130f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x8, #1, 129f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "tbz x8, #0, 130f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "st1 { v12.s }[2], [x25]\n"
       "st1 { v16.s }[2], [x24]\n"
       "st1 { v20.s }[2], [x23]\n"
       "b 130f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "str s12, [x25, #0x0]\n"
       "str s16, [x24, #0x0]\n"
       "str s20, [x23, #0x0]\n"
       "130:"  // Height 4: Partial direct writeback: Done
       "b 132f\n"
       "131:"  // Height 4: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -1986,22 +1986,22 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "bgt 101b\n"
       "b 200f\n"
       "133:"  // Height 5
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
       "134:"  // Height 5: Column loop
-      "cbz x16, 135f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 135f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -2016,48 +2016,48 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 146f\n"
       "135:"  // Height 5: no bias
       "tbz %x[flags], #0, 145f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x8, #0x10\n"
-      "add x25, x15, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 144f\n"
       "tbz x8, #3, 139f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x25], #0x10\n"
-      "ld1 { v17.4s }, [x24], #0x10\n"
       "ld1 { v20.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "ld1 { v21.4s }, [x23], #0x10\n"
       "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x8, #2, 137f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "ld1 { v14.4s }, [x25], #0x10\n"
       "ld1 { v18.4s }, [x24], #0x10\n"
       "ld1 { v22.4s }, [x23], #0x10\n"
       "ld1 { v26.4s }, [x22], #0x10\n"
       "tbz x8, #1, 136f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "ldr d27, [x22], #0x8\n"
       "tbz x8, #0, 143f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "ld1 { v15.s }[2], [x25]\n"
       "ld1 { v19.s }[2], [x24]\n"
       "ld1 { v23.s }[2], [x23]\n"
       "ld1 { v27.s }[2], [x22]\n"
       "b 143f\n"
       "136:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 143f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "ldr s15, [x25, #0x0]\n"
       "ldr s19, [x24, #0x0]\n"
       "ldr s23, [x23, #0x0]\n"
@@ -2065,23 +2065,23 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 143f\n"
       "137:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x8, #1, 138f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d14, [x25], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "ldr d26, [x22], #0x8\n"
       "tbz x8, #0, 143f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "ld1 { v14.s }[2], [x25]\n"
       "ld1 { v18.s }[2], [x24]\n"
       "ld1 { v22.s }[2], [x23]\n"
       "ld1 { v26.s }[2], [x22]\n"
       "b 143f\n"
       "138:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 143f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "ldr s14, [x25, #0x0]\n"
       "ldr s18, [x24, #0x0]\n"
       "ldr s22, [x23, #0x0]\n"
@@ -2089,29 +2089,29 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 143f\n"
       "139:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x8, #2, 141f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
       "ld1 { v20.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x8, #1, 140f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "tbz x8, #0, 143f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "ld1 { v13.s }[2], [x25]\n"
       "ld1 { v17.s }[2], [x24]\n"
       "ld1 { v21.s }[2], [x23]\n"
       "ld1 { v25.s }[2], [x22]\n"
       "b 143f\n"
       "140:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 143f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "ldr s13, [x25, #0x0]\n"
       "ldr s17, [x24, #0x0]\n"
       "ldr s21, [x23, #0x0]\n"
@@ -2119,34 +2119,34 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 143f\n"
       "141:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x8, #1, 142f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d12, [x25], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "ldr d24, [x22], #0x8\n"
       "tbz x8, #0, 143f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "ld1 { v12.s }[2], [x25]\n"
       "ld1 { v16.s }[2], [x24]\n"
       "ld1 { v20.s }[2], [x23]\n"
       "ld1 { v24.s }[2], [x22]\n"
       "b 143f\n"
       "142:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s12, [x25, #0x0]\n"
       "ldr s16, [x24, #0x0]\n"
       "ldr s20, [x23, #0x0]\n"
       "ldr s24, [x22, #0x0]\n"
       "143:"  // Height 5: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 146f\n"
       "144:"  // Height 5: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -2186,231 +2186,231 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "146:"  // Height 5: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "147:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 148f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "cbnz x14, 149f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
-      "add x28, x28, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "cbnz x15, 149f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
+      "add x9, x9, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 149f\n"
       "148:"  // Height 5: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #2\n"
-      "add x26, x28, x19, LSL #2\n"
-      "add x24, x26, x19, LSL #2\n"
-      "add x22, x24, x19, LSL #2\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #2\n"
+      "add x27, x9, x20, LSL #2\n"
+      "add x25, x27, x20, LSL #2\n"
+      "add x23, x25, x20, LSL #2\n"
       "149:"  // Height 5: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 152f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x8\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 151f\n"
       "150:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "add x28, x28, #0x10\n"
+      "ldr d6, [x17, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "ldr d6, [x17, #0x20]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "add x27, x27, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x17, #0x30]\n"
-      "add x26, x26, #0x10\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "ldr x10, [x17, #0x48]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x23, x23, #0x10\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "add x24, x24, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
       "ldr d6, [x17, #0x60]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "add x22, x22, #0x10\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "sub x14, x14, #0x4\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "cmp x14, #0x8\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "ldr x10, [x17, #0x88]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "sub x13, x13, #0x4\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "cmp x13, #0x8\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
-      "ldr x10, [x17, #0xa8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
       "ldr d7, [x17, #0xf0]\n"
-      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x100\n"
+      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "fmla v26.4s, v6.4s, v4.s[3]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
+      "ldr d4, [x23, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v7.d[1], x11\n"
       "bge 150b\n"
       "151:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x13, x13, #0x4\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "sub x14, x14, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
@@ -2491,17 +2491,17 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "152:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x13, 154f\n"
+      "cbz x14, 154f\n"
       "153:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
-      "ldr s1, [x28], #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
+      "ldr s1, [x9], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
@@ -2524,27 +2524,25 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "cbnz x13, 153b\n"
+      "cbnz x14, 153b\n"
       "154:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 147b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 155f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -2555,16 +2553,6 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
       "fmin v20.4s, v20.4s, v0.4s\n"
@@ -2575,22 +2563,34 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v25.4s, v25.4s, v0.4s\n"
       "fmin v26.4s, v26.4s, v0.4s\n"
       "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
       "155:"  // Height 5: No activation
       "cmp x8, #0x10\n"
       "bge 164f\n"
       "tbz x8, #3, 159f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v13.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
@@ -2600,19 +2600,19 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "st1 { v24.4s }, [x22], #0x10\n"
       "st1 { v25.4s }, [x22], #0x10\n"
       "tbz x8, #2, 157f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "st1 { v14.4s }, [x25], #0x10\n"
       "st1 { v18.4s }, [x24], #0x10\n"
       "st1 { v22.4s }, [x23], #0x10\n"
       "st1 { v26.4s }, [x22], #0x10\n"
       "tbz x8, #1, 156f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "str d27, [x22], #0x8\n"
       "tbz x8, #0, 163f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "st1 { v15.s }[2], [x25]\n"
       "st1 { v19.s }[2], [x24]\n"
       "st1 { v23.s }[2], [x23]\n"
@@ -2620,7 +2620,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "156:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x8, #0, 163f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "str s15, [x25, #0x0]\n"
       "str s19, [x24, #0x0]\n"
       "str s23, [x23, #0x0]\n"
@@ -2628,13 +2628,13 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "157:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x8, #1, 158f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "str d26, [x22], #0x8\n"
       "tbz x8, #0, 163f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "st1 { v14.s }[2], [x25]\n"
       "st1 { v18.s }[2], [x24]\n"
       "st1 { v22.s }[2], [x23]\n"
@@ -2642,7 +2642,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "158:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x8, #0, 163f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "str s14, [x25, #0x0]\n"
       "str s18, [x24, #0x0]\n"
       "str s22, [x23, #0x0]\n"
@@ -2650,19 +2650,19 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "159:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x8, #2, 161f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
       "st1 { v20.4s }, [x23], #0x10\n"
       "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x8, #1, 160f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "tbz x8, #0, 163f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "st1 { v13.s }[2], [x25]\n"
       "st1 { v17.s }[2], [x24]\n"
       "st1 { v21.s }[2], [x23]\n"
@@ -2670,7 +2670,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "160:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x8, #0, 163f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "str s13, [x25, #0x0]\n"
       "str s17, [x24, #0x0]\n"
       "str s21, [x23, #0x0]\n"
@@ -2678,20 +2678,20 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 163f\n"
       "161:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x8, #1, 162f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "tbz x8, #0, 163f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "st1 { v12.s }[2], [x25]\n"
       "st1 { v16.s }[2], [x24]\n"
       "st1 { v20.s }[2], [x23]\n"
       "st1 { v24.s }[2], [x22]\n"
       "b 163f\n"
       "162:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "str s12, [x25, #0x0]\n"
       "str s16, [x24, #0x0]\n"
       "str s20, [x23, #0x0]\n"
@@ -2699,11 +2699,11 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "163:"  // Height 5: Partial direct writeback: Done
       "b 165f\n"
       "164:"  // Height 5: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -2725,25 +2725,25 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "bgt 134b\n"
       "b 200f\n"
       "166:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x7, %x[bias]\n"
       "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[bias]\n"
       "ldr x17, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x16, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "167:"  // Height 6: Column loop
-      "cbz x16, 168f\n"
-      "ldr q8, [x16, #0x0]\n"
-      "ldr q9, [x16, #0x10]\n"
-      "ldr q10, [x16, #0x20]\n"
+      "cbz x7, 168f\n"
+      "ldr q8, [x7, #0x0]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q11, [x16, #0x30]\n"
+      "ldr q9, [x7, #0x10]\n"
       "mov v13.16b, v9.16b\n"
-      "add x16, x16, #0x40\n"
+      "ldr q10, [x7, #0x20]\n"
       "mov v14.16b, v10.16b\n"
+      "ldr q11, [x7, #0x30]\n"
       "mov v15.16b, v11.16b\n"
       "mov v16.16b, v8.16b\n"
+      "add x7, x7, #0x40\n"
       "mov v17.16b, v9.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
@@ -2762,44 +2762,44 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 179f\n"
       "168:"  // Height 6: no bias
       "tbz %x[flags], #0, 178f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x8, #0x10\n"
-      "add x25, x15, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 177f\n"
       "tbz x8, #3, 172f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x25], #0x10\n"
-      "ld1 { v17.4s }, [x24], #0x10\n"
       "ld1 { v20.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
       "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x16], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "ld1 { v21.4s }, [x23], #0x10\n"
       "ld1 { v25.4s }, [x22], #0x10\n"
       "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x8, #2, 170f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
+      "ld1 { v10.4s }, [x16], #0x10\n"
       "ld1 { v14.4s }, [x25], #0x10\n"
       "ld1 { v18.4s }, [x24], #0x10\n"
       "ld1 { v22.4s }, [x23], #0x10\n"
       "ld1 { v26.4s }, [x22], #0x10\n"
       "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x8, #1, 169f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x19, #0x38\n"
+      "ldr d11, [x16], #0x8\n"
+      "mov x20, #0x38\n"
       "ldr d15, [x25], #0x8\n"
       "ldr d19, [x24], #0x8\n"
       "ldr d23, [x23], #0x8\n"
       "ldr d27, [x22], #0x8\n"
       "ldr d31, [x21], #0x8\n"
       "tbz x8, #0, 176f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "ld1 { v11.s }[2], [x16]\n"
       "ld1 { v15.s }[2], [x25]\n"
       "ld1 { v19.s }[2], [x24]\n"
       "ld1 { v23.s }[2], [x23]\n"
@@ -2807,9 +2807,9 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "ld1 { v31.s }[2], [x21]\n"
       "b 176f\n"
       "169:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x8, #0, 176f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "ldr s11, [x16, #0x0]\n"
       "ldr s15, [x25, #0x0]\n"
       "ldr s19, [x24, #0x0]\n"
       "ldr s23, [x23, #0x0]\n"
@@ -2818,15 +2818,15 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 176f\n"
       "170:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x8, #1, 171f\n"
-      "ldr d10, [x15], #0x8\n"
+      "ldr d10, [x16], #0x8\n"
+      "mov x20, #0x28\n"
       "ldr d14, [x25], #0x8\n"
-      "mov x19, #0x28\n"
       "ldr d18, [x24], #0x8\n"
       "ldr d22, [x23], #0x8\n"
       "ldr d26, [x22], #0x8\n"
       "ldr d30, [x21], #0x8\n"
       "tbz x8, #0, 176f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "ld1 { v10.s }[2], [x16]\n"
       "ld1 { v14.s }[2], [x25]\n"
       "ld1 { v18.s }[2], [x24]\n"
       "ld1 { v22.s }[2], [x23]\n"
@@ -2834,9 +2834,9 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "ld1 { v30.s }[2], [x21]\n"
       "b 176f\n"
       "171:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x8, #0, 176f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "ldr s10, [x16, #0x0]\n"
       "ldr s14, [x25, #0x0]\n"
       "ldr s18, [x24, #0x0]\n"
       "ldr s22, [x23, #0x0]\n"
@@ -2845,22 +2845,22 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 176f\n"
       "172:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x8, #2, 174f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
+      "ld1 { v8.4s }, [x16], #0x10\n"
       "ld1 { v12.4s }, [x25], #0x10\n"
       "ld1 { v16.4s }, [x24], #0x10\n"
       "ld1 { v20.4s }, [x23], #0x10\n"
       "ld1 { v24.4s }, [x22], #0x10\n"
       "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x8, #1, 173f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x16], #0x8\n"
+      "mov x20, #0x18\n"
       "ldr d13, [x25], #0x8\n"
       "ldr d17, [x24], #0x8\n"
       "ldr d21, [x23], #0x8\n"
       "ldr d25, [x22], #0x8\n"
       "ldr d29, [x21], #0x8\n"
       "tbz x8, #0, 176f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "ld1 { v9.s }[2], [x16]\n"
       "ld1 { v13.s }[2], [x25]\n"
       "ld1 { v17.s }[2], [x24]\n"
       "ld1 { v21.s }[2], [x23]\n"
@@ -2868,9 +2868,9 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "ld1 { v29.s }[2], [x21]\n"
       "b 176f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x8, #0, 176f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "ldr s9, [x16, #0x0]\n"
       "ldr s13, [x25, #0x0]\n"
       "ldr s17, [x24, #0x0]\n"
       "ldr s21, [x23, #0x0]\n"
@@ -2879,15 +2879,15 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 176f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x8, #1, 175f\n"
-      "ldr d8, [x15], #0x8\n"
+      "ldr d8, [x16], #0x8\n"
+      "mov x20, #0x8\n"
       "ldr d12, [x25], #0x8\n"
-      "mov x19, #0x8\n"
       "ldr d16, [x24], #0x8\n"
       "ldr d20, [x23], #0x8\n"
       "ldr d24, [x22], #0x8\n"
       "ldr d28, [x21], #0x8\n"
       "tbz x8, #0, 176f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "ld1 { v8.s }[2], [x16]\n"
       "ld1 { v12.s }[2], [x25]\n"
       "ld1 { v16.s }[2], [x24]\n"
       "ld1 { v20.s }[2], [x23]\n"
@@ -2895,21 +2895,21 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "ld1 { v28.s }[2], [x21]\n"
       "b 176f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x16, #0x0]\n"
+      "mov x20, #0x0\n"
       "ldr s12, [x25, #0x0]\n"
       "ldr s16, [x24, #0x0]\n"
       "ldr s20, [x23, #0x0]\n"
       "ldr s24, [x22, #0x0]\n"
       "ldr s28, [x21, #0x0]\n"
       "176:"  // Height 6: Partial accumulate: Done
-      "sub x15, x15, x19\n"
+      "sub x16, x16, x20\n"
       "b 179f\n"
       "177:"  // Height 6: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x16, #0x0]\n"
+      "ldr q9, [x16, #0x10]\n"
+      "ldr q10, [x16, #0x20]\n"
+      "ldr q11, [x16, #0x30]\n"
       "ldr q12, [x25, #0x0]\n"
       "ldr q13, [x25, #0x10]\n"
       "ldr q14, [x25, #0x20]\n"
@@ -2957,260 +2957,260 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "179:"  // Height 6: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "180:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 181f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x14, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19, LSL #2\n"
-      "add x28, x28, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x15, 182f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20, LSL #2\n"
+      "add x9, x9, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 182f\n"
       "181:"  // Height 6: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19, LSL #2\n"
-      "add x26, x28, x19, LSL #2\n"
-      "add x24, x26, x19, LSL #2\n"
-      "add x22, x24, x19, LSL #2\n"
-      "add x20, x22, x19, LSL #2\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20, LSL #2\n"
+      "add x27, x9, x20, LSL #2\n"
+      "add x25, x27, x20, LSL #2\n"
+      "add x23, x25, x20, LSL #2\n"
+      "add x21, x23, x20, LSL #2\n"
       "182:"  // Height 6: input setup done
-      "cmp x13, #0x4\n"
+      "cmp x14, #0x4\n"
       "blt 185f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x8\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x8\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x17, #0x0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "blt 184f\n"
       "183:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr d7, [x17, #0x10]\n"
+      "ldr x12, [x17, #0x28]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x18]\n"
+      "ldr x11, [x17, #0x38]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "ldr x10, [x17, #0x28]\n"
+      "add x13, x13, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr d6, [x17, #0x20]\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "ldr x11, [x17, #0x38]\n"
+      "ldr x12, [x17, #0x48]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "ldr x10, [x17, #0x48]\n"
+      "add x21, x21, #0x10\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
       "ldr d7, [x17, #0x30]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x28, x28, #0x10\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x11, [x17, #0x58]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "ldr x11, [x17, #0x58]\n"
+      "ldr x28, [x9, #0x8]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x26, [x27, #0x8]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
       "ldr d6, [x17, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "add x26, x26, #0x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "ldr x12, [x17, #0x68]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "ldr x10, [x17, #0x68]\n"
+      "ldr x22, [x23, #0x8]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x21, #0x8]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
       "ldr d7, [x17, #0x50]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
-      "add x24, x24, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x17, #0x78]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x4\n"
       "fmla v20.4s, v6.4s, v3.s[1]\n"
-      "ldr x11, [x17, #0x78]\n"
+      "cmp x14, #0x8\n"
       "fmla v24.4s, v6.4s, v4.s[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v28.4s, v6.4s, v5.s[1]\n"
       "ldr d6, [x17, #0x60]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
-      "add x22, x22, #0x10\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr x12, [x17, #0x88]\n"
       "fmla v17.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[1]\n"
-      "ldr x10, [x17, #0x88]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v25.4s, v7.4s, v4.s[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v29.4s, v7.4s, v5.s[1]\n"
       "ldr d7, [x17, #0x70]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
-      "add x20, x20, #0x10\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "ldr x11, [x17, #0x98]\n"
       "fmla v18.4s, v6.4s, v2.s[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[1]\n"
-      "ldr x11, [x17, #0x98]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[1]\n"
-      "ldr x19, [x20, #0x8]\n"
       "fmla v30.4s, v6.4s, v5.s[1]\n"
       "ldr d6, [x17, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
-      "sub x13, x13, #0x4\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[1]\n"
-      "cmp x13, #0x8\n"
+      "ldr x12, [x17, #0xa8]\n"
       "fmla v19.4s, v7.4s, v2.s[1]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.4s, v7.4s, v3.s[1]\n"
-      "ldr x10, [x17, #0xa8]\n"
       "fmla v27.4s, v7.4s, v4.s[1]\n"
       "fmla v31.4s, v7.4s, v5.s[1]\n"
       "ldr d7, [x17, #0x90]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[2]\n"
       "fmla v12.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xb8]\n"
       "fmla v16.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.4s, v6.4s, v3.s[2]\n"
-      "ldr x11, [x17, #0xb8]\n"
       "fmla v24.4s, v6.4s, v4.s[2]\n"
       "fmla v28.4s, v6.4s, v5.s[2]\n"
       "ldr d6, [x17, #0xa0]\n"
       "fmla v9.4s, v7.4s, v0.s[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xc8]\n"
       "fmla v17.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.4s, v7.4s, v3.s[2]\n"
-      "ldr x10, [x17, #0xc8]\n"
       "fmla v25.4s, v7.4s, v4.s[2]\n"
       "fmla v29.4s, v7.4s, v5.s[2]\n"
       "ldr d7, [x17, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v10.4s, v6.4s, v0.s[2]\n"
       "fmla v14.4s, v6.4s, v1.s[2]\n"
+      "ldr x11, [x17, #0xd8]\n"
       "fmla v18.4s, v6.4s, v2.s[2]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.4s, v6.4s, v3.s[2]\n"
-      "ldr x11, [x17, #0xd8]\n"
       "fmla v26.4s, v6.4s, v4.s[2]\n"
       "fmla v30.4s, v6.4s, v5.s[2]\n"
       "ldr d6, [x17, #0xc0]\n"
       "fmla v11.4s, v7.4s, v0.s[2]\n"
+      "mov v6.d[1], x12\n"
       "fmla v15.4s, v7.4s, v1.s[2]\n"
+      "ldr x12, [x17, #0xe8]\n"
       "fmla v19.4s, v7.4s, v2.s[2]\n"
-      "mov v6.d[1], x10\n"
       "fmla v23.4s, v7.4s, v3.s[2]\n"
-      "ldr x10, [x17, #0xe8]\n"
       "fmla v27.4s, v7.4s, v4.s[2]\n"
       "fmla v31.4s, v7.4s, v5.s[2]\n"
       "ldr d7, [x17, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       "fmla v8.4s, v6.4s, v0.s[3]\n"
       "fmla v12.4s, v6.4s, v1.s[3]\n"
+      "ldr x11, [x17, #0xf8]\n"
       "fmla v16.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v20.4s, v6.4s, v3.s[3]\n"
-      "ldr x11, [x17, #0xf8]\n"
       "fmla v24.4s, v6.4s, v4.s[3]\n"
       "fmla v28.4s, v6.4s, v5.s[3]\n"
       "ldr d6, [x17, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
+      "mov v6.d[1], x12\n"
       "fmla v13.4s, v7.4s, v1.s[3]\n"
       "fmla v17.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
       "fmla v21.4s, v7.4s, v3.s[3]\n"
       "fmla v25.4s, v7.4s, v4.s[3]\n"
       "fmla v29.4s, v7.4s, v5.s[3]\n"
       "ldr d7, [x17, #0xf0]\n"
-      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "mov v7.d[1], x11\n"
       "add x17, x17, #0x100\n"
+      "fmla v10.4s, v6.4s, v0.s[3]\n"
+      "ldr x12, [x17, #0x8]\n"
       "fmla v14.4s, v6.4s, v1.s[3]\n"
-      "ldr x10, [x17, #0x8]\n"
+      "ldr x11, [x17, #0x18]\n"
       "fmla v18.4s, v6.4s, v2.s[3]\n"
-      "mov v7.d[1], x11\n"
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "fmla v26.4s, v6.4s, v4.s[3]\n"
       "fmla v30.4s, v6.4s, v5.s[3]\n"
       "ldr d6, [x17, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "mov v1.d[1], x27\n"
+      "ldr d4, [x23, #0x0]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "ldr d5, [x20, #0x0]\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
-      "mov v5.d[1], x19\n"
+      "ldr d5, [x21, #0x0]\n"
+      "ldr d7, [x17, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v5.d[1], x20\n"
+      "mov v7.d[1], x11\n"
       "bge 183b\n"
       "184:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x17, #0x10]\n"
+      "add x13, x13, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "sub x13, x13, #0x4\n"
+      "add x9, x9, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x17, #0x20]\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
+      "add x21, x21, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "add x26, x26, #0x10\n"
+      "sub x14, x14, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
       "ldr q7, [x17, #0x30]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
-      "add x20, x20, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
       "ldr q6, [x17, #0x40]\n"
@@ -3305,18 +3305,18 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
       "185:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x13, 187f\n"
+      "cbz x14, 187f\n"
       "186:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x1\n"
-      "ldr s1, [x28], #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x1\n"
+      "ldr s1, [x9], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x17, #0x0]\n"
-      "ldr q7, [x17, #0x10]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x17, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
@@ -3343,29 +3343,27 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
-      "cbnz x13, 186b\n"
+      "cbnz x14, 186b\n"
       "187:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 180b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "add x25, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x16, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x16, #0x0]\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 188f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v1.4s }, [x20]\n"
-      "ld1r { v0.4s }, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v0.4s }, [x20]\n"
       "fmin v8.4s, v8.4s, v0.4s\n"
       "fmin v9.4s, v9.4s, v0.4s\n"
       "fmin v10.4s, v10.4s, v0.4s\n"
@@ -3376,16 +3374,6 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v15.4s, v15.4s, v0.4s\n"
       "fmin v16.4s, v16.4s, v0.4s\n"
       "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
       "fmin v18.4s, v18.4s, v0.4s\n"
       "fmin v19.4s, v19.4s, v0.4s\n"
       "fmin v20.4s, v20.4s, v0.4s\n"
@@ -3396,30 +3384,42 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "fmin v25.4s, v25.4s, v0.4s\n"
       "fmin v26.4s, v26.4s, v0.4s\n"
       "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
       "fmin v28.4s, v28.4s, v0.4s\n"
       "fmin v29.4s, v29.4s, v0.4s\n"
       "fmin v30.4s, v30.4s, v0.4s\n"
       "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v31.4s, v31.4s, v0.4s\n"
       "188:"  // Height 6: No activation
       "cmp x8, #0x10\n"
       "bge 197f\n"
       "tbz x8, #3, 192f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
+      "st1 { v9.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v13.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
@@ -3431,21 +3431,21 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "st1 { v28.4s }, [x21], #0x10\n"
       "st1 { v29.4s }, [x21], #0x10\n"
       "tbz x8, #2, 190f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
+      "st1 { v10.4s }, [x16], #0x10\n"
       "st1 { v14.4s }, [x25], #0x10\n"
       "st1 { v18.4s }, [x24], #0x10\n"
       "st1 { v22.4s }, [x23], #0x10\n"
       "st1 { v26.4s }, [x22], #0x10\n"
       "st1 { v30.4s }, [x21], #0x10\n"
       "tbz x8, #1, 189f\n"
-      "str d11, [x15], #0x8\n"
+      "str d11, [x16], #0x8\n"
       "str d15, [x25], #0x8\n"
       "str d19, [x24], #0x8\n"
       "str d23, [x23], #0x8\n"
       "str d27, [x22], #0x8\n"
       "str d31, [x21], #0x8\n"
       "tbz x8, #0, 196f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "st1 { v11.s }[2], [x16]\n"
       "st1 { v15.s }[2], [x25]\n"
       "st1 { v19.s }[2], [x24]\n"
       "st1 { v23.s }[2], [x23]\n"
@@ -3454,7 +3454,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "189:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x8, #0, 196f\n"
-      "str s11, [x15, #0x0]\n"
+      "str s11, [x16, #0x0]\n"
       "str s15, [x25, #0x0]\n"
       "str s19, [x24, #0x0]\n"
       "str s23, [x23, #0x0]\n"
@@ -3463,14 +3463,14 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "190:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x8, #1, 191f\n"
-      "str d10, [x15], #0x8\n"
+      "str d10, [x16], #0x8\n"
       "str d14, [x25], #0x8\n"
       "str d18, [x24], #0x8\n"
       "str d22, [x23], #0x8\n"
       "str d26, [x22], #0x8\n"
       "str d30, [x21], #0x8\n"
       "tbz x8, #0, 196f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "st1 { v10.s }[2], [x16]\n"
       "st1 { v14.s }[2], [x25]\n"
       "st1 { v18.s }[2], [x24]\n"
       "st1 { v22.s }[2], [x23]\n"
@@ -3479,7 +3479,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "191:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x8, #0, 196f\n"
-      "str s10, [x15, #0x0]\n"
+      "str s10, [x16, #0x0]\n"
       "str s14, [x25, #0x0]\n"
       "str s18, [x24, #0x0]\n"
       "str s22, [x23, #0x0]\n"
@@ -3488,21 +3488,21 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "192:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x8, #2, 194f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
+      "st1 { v8.4s }, [x16], #0x10\n"
       "st1 { v12.4s }, [x25], #0x10\n"
       "st1 { v16.4s }, [x24], #0x10\n"
       "st1 { v20.4s }, [x23], #0x10\n"
       "st1 { v24.4s }, [x22], #0x10\n"
       "st1 { v28.4s }, [x21], #0x10\n"
       "tbz x8, #1, 193f\n"
-      "str d9, [x15], #0x8\n"
+      "str d9, [x16], #0x8\n"
       "str d13, [x25], #0x8\n"
       "str d17, [x24], #0x8\n"
       "str d21, [x23], #0x8\n"
       "str d25, [x22], #0x8\n"
       "str d29, [x21], #0x8\n"
       "tbz x8, #0, 196f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "st1 { v9.s }[2], [x16]\n"
       "st1 { v13.s }[2], [x25]\n"
       "st1 { v17.s }[2], [x24]\n"
       "st1 { v21.s }[2], [x23]\n"
@@ -3511,7 +3511,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "193:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x8, #0, 196f\n"
-      "str s9, [x15, #0x0]\n"
+      "str s9, [x16, #0x0]\n"
       "str s13, [x25, #0x0]\n"
       "str s17, [x24, #0x0]\n"
       "str s21, [x23, #0x0]\n"
@@ -3520,14 +3520,14 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "b 196f\n"
       "194:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x8, #1, 195f\n"
-      "str d8, [x15], #0x8\n"
+      "str d8, [x16], #0x8\n"
       "str d12, [x25], #0x8\n"
       "str d16, [x24], #0x8\n"
       "str d20, [x23], #0x8\n"
       "str d24, [x22], #0x8\n"
       "str d28, [x21], #0x8\n"
       "tbz x8, #0, 196f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "st1 { v8.s }[2], [x16]\n"
       "st1 { v12.s }[2], [x25]\n"
       "st1 { v16.s }[2], [x24]\n"
       "st1 { v20.s }[2], [x23]\n"
@@ -3535,7 +3535,7 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "st1 { v28.s }[2], [x21]\n"
       "b 196f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x16, #0x0]\n"
       "str s12, [x25, #0x0]\n"
       "str s16, [x24, #0x0]\n"
       "str s20, [x23, #0x0]\n"
@@ -3544,11 +3544,11 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "196:"  // Height 6: Partial direct writeback: Done
       "b 198f\n"
       "197:"  // Height 6: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x16, #0x0]\n"
+      "str q9, [x16, #0x10]\n"
+      "str q10, [x16, #0x20]\n"
+      "str q11, [x16, #0x30]\n"
+      "add x16, x16, #0x40\n"
       "str q12, [x25, #0x0]\n"
       "str q13, [x25, #0x10]\n"
       "str q14, [x25, #0x20]\n"
@@ -3574,20 +3574,20 @@ void a64_hybrid_fp32_mla_6x16_a55 (
       "bgt 167b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 200f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 199f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "199:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "200:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp
index 28e9be4cb7..c5e4388aa9 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -102,82 +102,82 @@ void a64_hybrid_fp32_mla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 67f\n"
       "beq 34f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x9, 3f\n"
-      "ldr q8, [x9, #0x0]\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "cbz x12, 3f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "b 14f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 13f\n"
       "cmp x11, #0x10\n"
       "bge 12f\n"
       "tbz x11, #3, 7f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 5f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 4f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 11f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 11f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 11f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x11, #1, 6f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 11f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 11f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 11f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x11, #2, 9f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 8f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 11f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 11f\n"
-      "ldr s9, [x28, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
       "b 11f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x11, #1, 10f\n"
-      "ldr d8, [x28], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v8.s }[2], [x28]\n"
+      "ld1 { v8.s }[2], [x9]\n"
       "b 11f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s8, [x9, #0x0]\n"
+      "mov x20, #0x0\n"
       "11:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 14f\n"
       "12:"  // Height 1: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
       "b 14f\n"
       "13:"  // Height 1: no accumulate
       "movi v8.16b, #0x0\n"
@@ -185,46 +185,42 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
       "14:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 17f\n"
       "16:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "17:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 20f\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 19f\n"
       "18:"  // Height 1: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x4\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x8\n"
-      "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x10, #0x60]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x10, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x10, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[1]\n"
@@ -241,27 +237,28 @@ void a64_hybrid_fp32_mla_6x16 (
       "ldr q6, [x10, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
-      "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "add x10, x10, #0x100\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 18b\n"
       "19:"  // Height 1: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
-      "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "fmla v10.4s, v6.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "ldr q6, [x10, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "ldr q7, [x10, #0x70]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
@@ -280,205 +277,208 @@ void a64_hybrid_fp32_mla_6x16 (
       "ldr q6, [x10, #0xe0]\n"
       "fmla v9.4s, v7.4s, v0.s[3]\n"
       "ldr q7, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x4\n"
       "fmla v10.4s, v6.4s, v0.s[3]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "20:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 22f\n"
+      "cbz x27, 22f\n"
       "21:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
+      "ldr s0, [x26], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
+      "sub x27, x27, #0x1\n"
       "ldr q7, [x10, #0x10]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
+      "ldr q7, [x10, #0x30]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
-      "cbnz x26, 21b\n"
+      "add x10, x10, #0x40\n"
+      "cbnz x27, 21b\n"
       "22:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 15b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "tbz %x[flags], #1, 23f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
       "23:"  // Height 1: No activation
       "cmp x11, #0x10\n"
       "bge 32f\n"
       "tbz x11, #3, 27f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 25f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 24f\n"
-      "str d11, [x28], #0x8\n"
+      "str d11, [x9], #0x8\n"
       "tbz x11, #0, 31f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 31f\n"
       "24:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x11, #0, 31f\n"
-      "str s11, [x28, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
       "b 31f\n"
       "25:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x11, #1, 26f\n"
-      "str d10, [x28], #0x8\n"
+      "str d10, [x9], #0x8\n"
       "tbz x11, #0, 31f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 31f\n"
       "26:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x11, #0, 31f\n"
-      "str s10, [x28, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
       "b 31f\n"
       "27:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x11, #2, 29f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 28f\n"
-      "str d9, [x28], #0x8\n"
+      "str d9, [x9], #0x8\n"
       "tbz x11, #0, 31f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 31f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x11, #0, 31f\n"
-      "str s9, [x28, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
       "b 31f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x11, #1, 30f\n"
-      "str d8, [x28], #0x8\n"
+      "str d8, [x9], #0x8\n"
       "tbz x11, #0, 31f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 31f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "31:"  // Height 1: Partial direct writeback: Done
       "b 33f\n"
       "32:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "33:"  // Height 1: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 200f\n"
       "34:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "35:"  // Height 2: Column loop
-      "cbz x9, 36f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 36f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "ldr q10, [x9, #0x20]\n"
       "mov v13.16b, v9.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "add x12, x12, #0x40\n"
       "b 47f\n"
       "36:"  // Height 2: no bias
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "bge 45f\n"
       "tbz x11, #3, 40f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 38f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 37f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 44f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 44f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x11, #1, 39f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 44f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 44f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x11, #2, 42f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 41f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 44f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 44f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x11, #1, 43f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 44f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "b 47f\n"
       "46:"  // Height 2: no accumulate
       "movi v8.16b, #0x0\n"
@@ -490,58 +490,58 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "50:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 53f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x4\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "cmp x26, #0x8\n"
+      "add x26, x26, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "add x25, x25, #0x10\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "cmp x27, #0x8\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "ldr q7, [x10, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[1]\n"
       "fmla v14.4s, v6.4s, v1.s[1]\n"
       "ldr q6, [x10, #0x80]\n"
@@ -571,32 +571,32 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v14.4s, v6.4s, v1.s[3]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x25, x25, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "sub x27, x27, #0x4\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v9.4s, v7.4s, v0.s[1]\n"
       "fmla v13.4s, v7.4s, v1.s[1]\n"
       "ldr q7, [x10, #0x70]\n"
@@ -630,262 +630,262 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v11.4s, v7.4s, v0.s[3]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 55f\n"
+      "cbz x27, 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "add x10, x10, #0x40\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
-      "cbnz x26, 54b\n"
+      "cbnz x27, 54b\n"
       "55:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 56f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
       "56:"  // Height 2: No activation
       "cmp x11, #0x10\n"
       "bge 65f\n"
       "tbz x11, #3, 60f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 58f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 57f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
       "tbz x11, #0, 64f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
       "b 64f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x11, #0, 64f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
       "b 64f\n"
       "58:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x11, #1, 59f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
       "tbz x11, #0, 64f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
       "b 64f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x11, #0, 64f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
       "b 64f\n"
       "60:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x11, #2, 62f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 61f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
       "tbz x11, #0, 64f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
       "b 64f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x11, #0, 64f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
       "b 64f\n"
       "62:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x11, #1, 63f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
       "tbz x11, #0, 64f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
       "b 64f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
       "64:"  // Height 2: Partial direct writeback: Done
       "b 66f\n"
       "65:"  // Height 2: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
       "66:"  // Height 2: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 35b\n"
       "b 200f\n"
       "67:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "68:"  // Height 3: Column loop
-      "cbz x9, 69f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 69f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
       "mov v13.16b, v9.16b\n"
-      "add x9, x9, #0x40\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
       "b 80f\n"
       "69:"  // Height 3: no bias
       "tbz %x[flags], #0, 79f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 78f\n"
       "tbz x11, #3, 73f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 71f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 70f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
       "tbz x11, #0, 77f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 77f\n"
       "70:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 77f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 77f\n"
       "71:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x11, #1, 72f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
       "tbz x11, #0, 77f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 77f\n"
       "72:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 77f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 77f\n"
       "73:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x11, #2, 75f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 74f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
       "tbz x11, #0, 77f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 77f\n"
       "74:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 77f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
       "b 77f\n"
       "75:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x11, #1, 76f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
       "tbz x11, #0, 77f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
       "b 77f\n"
       "76:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
       "77:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 80f\n"
       "78:"  // Height 3: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
       "b 80f\n"
       "79:"  // Height 3: no accumulate
       "movi v8.16b, #0x0\n"
@@ -901,62 +901,62 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "80:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "81:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 82f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 83f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 83f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 83f\n"
       "82:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "83:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 86f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 85f\n"
       "84:"  // Height 3: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x4\n"
+      "add x25, x25, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x8\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "add x24, x24, #0x10\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "cmp x27, #0x8\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v8.4s, v6.4s, v0.s[1]\n"
       "fmla v12.4s, v6.4s, v1.s[1]\n"
       "fmla v16.4s, v6.4s, v2.s[1]\n"
@@ -1003,34 +1003,34 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v18.4s, v6.4s, v2.s[3]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 84b\n"
       "85:"  // Height 3: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x24, x24, #0x10\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
+      "add x24, x24, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       "ldr q7, [x10, #0x30]\n"
+      "sub x27, x27, #0x4\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -1082,16 +1082,16 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v15.4s, v7.4s, v1.s[3]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "86:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 88f\n"
+      "cbz x27, 88f\n"
       "87:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x23], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
@@ -1105,299 +1105,299 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
-      "cbnz x26, 87b\n"
+      "cbnz x27, 87b\n"
       "88:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 81b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 89f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "89:"  // Height 3: No activation
       "cmp x11, #0x10\n"
       "bge 98f\n"
       "tbz x11, #3, 93f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 91f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 90f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
       "tbz x11, #0, 97f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 97f\n"
       "90:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x11, #0, 97f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 97f\n"
       "91:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x11, #1, 92f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
       "tbz x11, #0, 97f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 97f\n"
       "92:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x11, #0, 97f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 97f\n"
       "93:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x11, #2, 95f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 94f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
       "tbz x11, #0, 97f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 97f\n"
       "94:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x11, #0, 97f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 97f\n"
       "95:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x11, #1, 96f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
       "tbz x11, #0, 97f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 97f\n"
       "96:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "97:"  // Height 3: Partial direct writeback: Done
       "b 99f\n"
       "98:"  // Height 3: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "99:"  // Height 3: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 68b\n"
       "b 200f\n"
       "100:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "101:"  // Height 4: Column loop
-      "cbz x9, 102f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 102f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
       "b 113f\n"
       "102:"  // Height 4: no bias
       "tbz %x[flags], #0, 112f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 111f\n"
       "tbz x11, #3, 106f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 104f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 103f\n"
-      "mov x19, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
       "tbz x11, #0, 110f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 110f\n"
       "103:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 110f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 110f\n"
       "104:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x11, #1, 105f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
       "tbz x11, #0, 110f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 110f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 110f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 110f\n"
       "106:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x11, #2, 108f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 107f\n"
-      "mov x19, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
       "tbz x11, #0, 110f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 110f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 110f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 110f\n"
       "108:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x11, #1, 109f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
       "tbz x11, #0, 110f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 110f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "110:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 113f\n"
       "111:"  // Height 4: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "b 113f\n"
       "112:"  // Height 4: no accumulate
       "movi v8.16b, #0x0\n"
@@ -1417,69 +1417,69 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "113:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "114:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 115f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 116f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 116f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 116f\n"
       "115:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "116:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 119f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 118f\n"
       "117:"  // Height 4: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x8\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "cmp x27, #0x8\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x10, #0x50]\n"
@@ -1540,40 +1540,40 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v22.4s, v6.4s, v3.s[3]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 117b\n"
       "118:"  // Height 4: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
@@ -1639,16 +1639,16 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v19.4s, v7.4s, v2.s[3]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "119:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 121f\n"
+      "cbz x27, 121f\n"
       "120:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
@@ -1667,354 +1667,354 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v15.4s, v7.4s, v1.s[0]\n"
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
-      "cbnz x26, 120b\n"
+      "cbnz x27, 120b\n"
       "121:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 114b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 122f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
       "122:"  // Height 4: No activation
       "cmp x11, #0x10\n"
       "bge 131f\n"
       "tbz x11, #3, 126f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 124f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 123f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
       "tbz x11, #0, 130f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
       "b 130f\n"
       "123:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x11, #0, 130f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
       "b 130f\n"
       "124:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x11, #1, 125f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
       "tbz x11, #0, 130f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
       "b 130f\n"
       "125:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x11, #0, 130f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
       "b 130f\n"
       "126:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x11, #2, 128f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 127f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
       "tbz x11, #0, 130f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
       "b 130f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x11, #0, 130f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
       "b 130f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x11, #1, 129f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
       "tbz x11, #0, 130f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
       "b 130f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
       "130:"  // Height 4: Partial direct writeback: Done
       "b 132f\n"
       "131:"  // Height 4: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
       "132:"  // Height 4: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 101b\n"
       "b 200f\n"
       "133:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "134:"  // Height 5: Column loop
-      "cbz x9, 135f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 135f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
-      "mov v17.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
       "b 146f\n"
       "135:"  // Height 5: no bias
       "tbz %x[flags], #0, 145f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 144f\n"
       "tbz x11, #3, 139f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 137f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 136f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
       "tbz x11, #0, 143f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 143f\n"
       "136:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 143f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 143f\n"
       "137:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x11, #1, 138f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
       "tbz x11, #0, 143f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 143f\n"
       "138:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 143f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 143f\n"
       "139:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x11, #2, 141f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 140f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x11, #0, 143f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 143f\n"
       "140:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 143f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 143f\n"
       "141:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x11, #1, 142f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
       "tbz x11, #0, 143f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 143f\n"
       "142:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "143:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 146f\n"
       "144:"  // Height 5: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
       "b 146f\n"
       "145:"  // Height 5: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2038,74 +2038,74 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "146:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "147:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 148f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 149f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 149f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 149f\n"
       "148:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "149:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 152f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 151f\n"
       "150:"  // Height 5: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x4\n"
+      "add x23, x23, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x8\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x8\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
@@ -2182,45 +2182,45 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v26.4s, v6.4s, v4.s[3]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 150b\n"
       "151:"  // Height 5: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
+      "add x22, x22, #0x10\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x4\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
       "fmla v11.4s, v7.4s, v0.s[0]\n"
@@ -2301,18 +2301,18 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v23.4s, v7.4s, v3.s[3]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "152:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 154f\n"
+      "cbz x27, 154f\n"
       "153:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
@@ -2334,412 +2334,412 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v19.4s, v7.4s, v2.s[0]\n"
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
-      "cbnz x26, 153b\n"
+      "cbnz x27, 153b\n"
       "154:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 147b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 155f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
       "155:"  // Height 5: No activation
       "cmp x11, #0x10\n"
       "bge 164f\n"
       "tbz x11, #3, 159f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 157f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 156f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
       "tbz x11, #0, 163f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 163f\n"
       "156:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x11, #0, 163f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 163f\n"
       "157:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x11, #1, 158f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
       "tbz x11, #0, 163f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 163f\n"
       "158:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x11, #0, 163f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 163f\n"
       "159:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x11, #2, 161f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 160f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x11, #0, 163f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 163f\n"
       "160:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x11, #0, 163f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 163f\n"
       "161:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x11, #1, 162f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x11, #0, 163f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 163f\n"
       "162:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "163:"  // Height 5: Partial direct writeback: Done
       "b 165f\n"
       "164:"  // Height 5: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "165:"  // Height 5: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 134b\n"
       "b 200f\n"
       "166:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "167:"  // Height 6: Column loop
-      "cbz x9, 168f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 168f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "mov v12.16b, v8.16b\n"
-      "ldr q9, [x9, #0x10]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v24.16b, v8.16b\n"
-      "mov v28.16b, v8.16b\n"
       "mov v13.16b, v9.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "mov v14.16b, v10.16b\n"
       "mov v15.16b, v11.16b\n"
+      "mov v16.16b, v8.16b\n"
       "mov v17.16b, v9.16b\n"
+      "add x12, x12, #0x40\n"
       "mov v18.16b, v10.16b\n"
       "mov v19.16b, v11.16b\n"
+      "mov v20.16b, v8.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v22.16b, v10.16b\n"
       "mov v23.16b, v11.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v26.16b, v10.16b\n"
       "mov v27.16b, v11.16b\n"
+      "mov v28.16b, v8.16b\n"
       "mov v29.16b, v9.16b\n"
       "mov v30.16b, v10.16b\n"
       "mov v31.16b, v11.16b\n"
       "b 179f\n"
       "168:"  // Height 6: no bias
       "tbz %x[flags], #0, 178f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 177f\n"
       "tbz x11, #3, 172f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 170f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 169f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x11, #0, 176f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 176f\n"
       "169:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 176f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 176f\n"
       "170:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x11, #1, 171f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x11, #0, 176f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 176f\n"
       "171:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 176f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 176f\n"
       "172:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x11, #2, 174f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v16.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v16.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 173f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x11, #0, 176f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 176f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 176f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 176f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x11, #1, 175f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d16, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d16, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x11, #0, 176f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v16.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v16.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 176f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s16, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s16, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "176:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 179f\n"
       "177:"  // Height 6: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q16, [x23, #0x0]\n"
-      "ldr q17, [x23, #0x10]\n"
-      "ldr q18, [x23, #0x20]\n"
-      "ldr q19, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q24, [x21, #0x0]\n"
-      "ldr q25, [x21, #0x10]\n"
-      "ldr q26, [x21, #0x20]\n"
-      "ldr q27, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q16, [x24, #0x0]\n"
+      "ldr q17, [x24, #0x10]\n"
+      "ldr q18, [x24, #0x20]\n"
+      "ldr q19, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q24, [x22, #0x0]\n"
+      "ldr q25, [x22, #0x10]\n"
+      "ldr q26, [x22, #0x20]\n"
+      "ldr q27, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "b 179f\n"
       "178:"  // Height 6: no accumulate
       "movi v8.16b, #0x0\n"
@@ -2767,82 +2767,82 @@ void a64_hybrid_fp32_mla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "179:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "180:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 181f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 182f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 182f\n"
       "181:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "182:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 185f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x8\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 184f\n"
       "183:"  // Height 6: Multiply loop: Main loop head
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x23, x23, #0x10\n"
       "fmla v9.4s, v7.4s, v0.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x8\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
+      "cmp x27, #0x8\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -2932,51 +2932,51 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v30.4s, v6.4s, v5.s[3]\n"
       "ldr q6, [x10, #0x0]\n"
       "fmla v11.4s, v7.4s, v0.s[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       "fmla v15.4s, v7.4s, v1.s[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       "fmla v19.4s, v7.4s, v2.s[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       "fmla v23.4s, v7.4s, v3.s[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       "fmla v27.4s, v7.4s, v4.s[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 183b\n"
       "184:"  // Height 6: Multiply loop: Single iteration only
       "fmla v8.4s, v6.4s, v0.s[0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      "sub x26, x26, #0x4\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
-      "add x24, x24, #0x10\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v24.4s, v6.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       "fmla v28.4s, v6.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
-      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "add x22, x22, #0x10\n"
+      "fmla v9.4s, v7.4s, v0.s[0]\n"
       "fmla v13.4s, v7.4s, v1.s[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x4\n"
       "fmla v17.4s, v7.4s, v2.s[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v21.4s, v7.4s, v3.s[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v25.4s, v7.4s, v4.s[0]\n"
       "fmla v29.4s, v7.4s, v5.s[0]\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v10.4s, v6.4s, v0.s[0]\n"
       "fmla v14.4s, v6.4s, v1.s[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v18.4s, v6.4s, v2.s[0]\n"
       "fmla v22.4s, v6.4s, v3.s[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v26.4s, v6.4s, v4.s[0]\n"
       "fmla v30.4s, v6.4s, v5.s[0]\n"
       "ldr q6, [x10, #0x40]\n"
@@ -3071,18 +3071,18 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v27.4s, v7.4s, v4.s[3]\n"
       "fmla v31.4s, v7.4s, v5.s[3]\n"
       "185:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 187f\n"
+      "cbz x27, 187f\n"
       "186:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x1\n"
-      "ldr s1, [x24], #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x1\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x10, #0x0]\n"
-      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "ldr q7, [x10, #0x10]\n"
+      "fmla v8.4s, v6.4s, v0.s[0]\n"
       "fmla v12.4s, v6.4s, v1.s[0]\n"
       "fmla v16.4s, v6.4s, v2.s[0]\n"
       "fmla v20.4s, v6.4s, v3.s[0]\n"
@@ -3109,251 +3109,251 @@ void a64_hybrid_fp32_mla_6x16 (
       "fmla v23.4s, v7.4s, v3.s[0]\n"
       "fmla v27.4s, v7.4s, v4.s[0]\n"
       "fmla v31.4s, v7.4s, v5.s[0]\n"
-      "cbnz x26, 186b\n"
+      "cbnz x27, 186b\n"
       "187:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 180b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbz %x[flags], #1, 188f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v31.4s, v31.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v31.4s, v31.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v31.4s, v31.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v31.4s, v31.4s, v0.4s\n"
       "188:"  // Height 6: No activation
       "cmp x11, #0x10\n"
       "bge 197f\n"
       "tbz x11, #3, 192f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v13.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v13.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 190f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
-      "st1 { v22.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v30.4s }, [x20], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
+      "st1 { v22.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 189f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
-      "str d23, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
+      "str d23, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
-      "st1 { v23.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
-      "st1 { v31.s }[2], [x20]\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
+      "st1 { v23.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
+      "st1 { v31.s }[2], [x21]\n"
       "b 196f\n"
       "189:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x11, #0, 196f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
-      "str s23, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
-      "str s31, [x20, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
+      "str s23, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
+      "str s31, [x21, #0x0]\n"
       "b 196f\n"
       "190:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x11, #1, 191f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
       "b 196f\n"
       "191:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x11, #0, 196f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
       "b 196f\n"
       "192:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x11, #2, 194f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 193f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
       "b 196f\n"
       "193:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x11, #0, 196f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
       "b 196f\n"
       "194:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x11, #1, 195f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
       "tbz x11, #0, 196f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
       "b 196f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
       "196:"  // Height 6: Partial direct writeback: Done
       "b 198f\n"
       "197:"  // Height 6: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x24, #0x0]\n"
-      "str q13, [x24, #0x10]\n"
-      "str q14, [x24, #0x20]\n"
-      "str q15, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
-      "str q20, [x22, #0x0]\n"
-      "str q21, [x22, #0x10]\n"
-      "str q22, [x22, #0x20]\n"
-      "str q23, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
-      "str q28, [x20, #0x0]\n"
-      "str q29, [x20, #0x10]\n"
-      "str q30, [x20, #0x20]\n"
-      "str q31, [x20, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x25, #0x0]\n"
+      "str q13, [x25, #0x10]\n"
+      "str q14, [x25, #0x20]\n"
+      "str q15, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
+      "str q20, [x23, #0x0]\n"
+      "str q21, [x23, #0x10]\n"
+      "str q22, [x23, #0x20]\n"
+      "str q23, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
+      "str q28, [x21, #0x0]\n"
+      "str q29, [x21, #0x10]\n"
+      "str q30, [x21, #0x20]\n"
+      "str q31, [x21, #0x30]\n"
       "198:"  // Height 6: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 167b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 200f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 199f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "199:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "200:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp
index 99920002b2..e2dae4b414 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -120,15 +120,15 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "bge 6f\n"
       "tbz x17, #1, 4f\n"
       "ldr d24, [x14], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "tbz x17, #0, 5f\n"
       "ld1 { v24.s }[2], [x14]\n"
       "b 5f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "5:"  // Height 1: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 8f\n"
       "6:"  // Height 1: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -139,15 +139,15 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "9:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 10f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "cbnz x13, 11f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
       "b 11f\n"
       "10:"  // Height 1: setup direct input
       "mov x11, %x[input_ptr]\n"
@@ -161,20 +161,20 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "12:"  // Height 1: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "add x11, x11, #0x10\n"
       "ldr d10, [x16, #0x20]\n"
       "sub x12, x12, #0x4\n"
       "ldr x21, [x16, #0x28]\n"
       "cmp x12, #0x8\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "ldr d11, [x16, #0x30]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "add x16, x16, #0x40\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "prfm pldl1keep, [x11, #0x80]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "ldr d8, [x16, #0x0]\n"
       "ldr x26, [x16, #0x8]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
@@ -208,14 +208,14 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "16:"  // Height 1: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 9b\n"
       "prfm pstl1keep, [x14, #0x0]\n"
       "tbz %x[flags], #1, 17f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "add x8, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v17.4s }, [x8]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmax v24.4s, v24.4s, v17.4s\n"
       "17:"  // Height 1: No activation
@@ -250,24 +250,24 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 29f\n"
       "24:"  // Height 2: no bias
       "tbz %x[flags], #0, 28f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "bge 27f\n"
       "tbz x17, #1, 25f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "tbz x17, #0, 26f\n"
       "ld1 { v24.s }[2], [x14]\n"
       "ld1 { v25.s }[2], [x27]\n"
       "b 26f\n"
       "25:"  // Height 2: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "26:"  // Height 2: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 29f\n"
       "27:"  // Height 2: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -280,21 +280,21 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "30:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 31f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "cbnz x13, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
       "b 32f\n"
       "31:"  // Height 2: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
       "32:"  // Height 2: input setup done
       "cmp x12, #0x4\n"
       "blt 35f\n"
@@ -307,20 +307,20 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "ldr d10, [x16, #0x20]\n"
       "add x11, x11, #0x10\n"
       "ldr x21, [x16, #0x28]\n"
       "add x9, x9, #0x10\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "ldr d11, [x16, #0x30]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "sub x12, x12, #0x4\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "prfm pldl1keep, [x9, #0x80]\n"
       "ldr x10, [x11, #0x8]\n"
       "cmp x12, #0x8\n"
@@ -370,17 +370,17 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "37:"  // Height 2: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
       "tbz %x[flags], #1, 38f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmax v24.4s, v24.4s, v17.4s\n"
@@ -422,15 +422,15 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 50f\n"
       "45:"  // Height 3: no bias
       "tbz %x[flags], #0, 49f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "bge 48f\n"
       "tbz x17, #1, 46f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "tbz x17, #0, 47f\n"
       "ld1 { v24.s }[2], [x14]\n"
@@ -439,11 +439,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 47f\n"
       "46:"  // Height 3: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "47:"  // Height 3: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 50f\n"
       "48:"  // Height 3: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -458,24 +458,24 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "51:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 52f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
       "cbnz x13, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
       "b 53f\n"
       "52:"  // Height 3: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
       "53:"  // Height 3: input setup done
       "cmp x12, #0x4\n"
       "blt 56f\n"
@@ -489,21 +489,21 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "ldr x21, [x16, #0x28]\n"
       "add x11, x11, #0x10\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "ldr d11, [x16, #0x30]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "add x9, x9, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "prfm pldl1keep, [x9, #0x80]\n"
       "add x27, x27, #0x10\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
@@ -569,19 +569,19 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "58:"  // Height 3: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 51b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
       "tbz %x[flags], #1, 59f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -630,16 +630,16 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 71f\n"
       "66:"  // Height 4: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "bge 69f\n"
       "tbz x17, #1, 67f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "ldr d27, [x25], #0x8\n"
       "tbz x17, #0, 68f\n"
@@ -650,12 +650,12 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 68f\n"
       "67:"  // Height 4: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "ldr s27, [x25, #0x0]\n"
       "68:"  // Height 4: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 71f\n"
       "69:"  // Height 4: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -672,27 +672,27 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "72:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 73f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
       "ldr x25, [x20, #0x18]\n"
       "cbnz x13, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
+      "add x25, x25, x8, LSL #2\n"
       "b 74f\n"
       "73:"  // Height 4: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
-      "add x25, x27, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
+      "add x25, x27, x8, LSL #2\n"
       "74:"  // Height 4: input setup done
       "cmp x12, #0x4\n"
       "blt 77f\n"
@@ -707,21 +707,21 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
       "ldr x21, [x16, #0x28]\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "ldr d11, [x16, #0x30]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "add x11, x11, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "ldr x10, [x11, #0x8]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
@@ -748,11 +748,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "sub x12, x12, #0x4\n"
       "mov v2.d[1], x26\n"
       "ldr d3, [x25, #0x0]\n"
-      "ldr x19, [x25, #0x8]\n"
+      "ldr x8, [x25, #0x8]\n"
       "cmp x12, #0x8\n"
       "add x16, x16, #0x40\n"
       "ldr d8, [x16, #0x0]\n"
-      "mov v3.d[1], x19\n"
+      "mov v3.d[1], x8\n"
       "ldr x26, [x16, #0x8]\n"
       "mov v8.d[1], x26\n"
       "bge 75b\n"
@@ -804,21 +804,21 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "79:"  // Height 4: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 80f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -874,17 +874,17 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 92f\n"
       "87:"  // Height 5: no bias
       "tbz %x[flags], #0, 91f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "bge 90f\n"
       "tbz x17, #1, 88f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "ldr d27, [x25], #0x8\n"
       "ldr d28, [x24], #0x8\n"
@@ -897,13 +897,13 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 89f\n"
       "88:"  // Height 5: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "ldr s27, [x25, #0x0]\n"
       "ldr s28, [x24, #0x0]\n"
       "89:"  // Height 5: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 92f\n"
       "90:"  // Height 5: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -922,30 +922,30 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "93:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 94f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
       "ldr x25, [x20, #0x18]\n"
       "ldr x24, [x20, #0x20]\n"
       "cbnz x13, 95f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
+      "add x25, x25, x8, LSL #2\n"
+      "add x24, x24, x8, LSL #2\n"
       "b 95f\n"
       "94:"  // Height 5: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
-      "add x25, x27, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
+      "add x25, x27, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "95:"  // Height 5: input setup done
       "cmp x12, #0x4\n"
       "blt 98f\n"
@@ -961,25 +961,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
       "ldr x21, [x16, #0x28]\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "ldr d11, [x16, #0x30]\n"
       "add x11, x11, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "ldr x10, [x11, #0x8]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "add x9, x9, #0x10\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
@@ -1008,12 +1008,12 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "sub x12, x12, #0x4\n"
       "mov v2.d[1], x26\n"
       "ldr d3, [x25, #0x0]\n"
-      "ldr x19, [x25, #0x8]\n"
+      "ldr x8, [x25, #0x8]\n"
       "cmp x12, #0x8\n"
       "ldr d4, [x24, #0x0]\n"
       "add x16, x16, #0x40\n"
       "ldr x21, [x24, #0x8]\n"
-      "mov v3.d[1], x19\n"
+      "mov v3.d[1], x8\n"
       "ldr d8, [x16, #0x0]\n"
       "ldr x26, [x16, #0x8]\n"
       "mov v4.d[1], x21\n"
@@ -1075,23 +1075,23 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "100:"  // Height 5: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 93b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
       "tbz %x[flags], #1, 101f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -1154,18 +1154,18 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 113f\n"
       "108:"  // Height 6: no bias
       "tbz %x[flags], #0, 112f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
       "bge 111f\n"
       "tbz x17, #1, 109f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "ldr d27, [x25], #0x8\n"
       "ldr d28, [x24], #0x8\n"
@@ -1180,14 +1180,14 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 110f\n"
       "109:"  // Height 6: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "ldr s27, [x25, #0x0]\n"
       "ldr s28, [x24, #0x0]\n"
       "ldr s29, [x23, #0x0]\n"
       "110:"  // Height 6: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 113f\n"
       "111:"  // Height 6: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -1208,11 +1208,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "114:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 115f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
@@ -1220,21 +1220,21 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr x24, [x20, #0x20]\n"
       "ldr x23, [x20, #0x28]\n"
       "cbnz x13, 116f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
+      "add x25, x25, x8, LSL #2\n"
+      "add x24, x24, x8, LSL #2\n"
+      "add x23, x23, x8, LSL #2\n"
       "b 116f\n"
       "115:"  // Height 6: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
-      "add x25, x27, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
+      "add x25, x27, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
       "116:"  // Height 6: input setup done
       "cmp x12, #0x4\n"
       "blt 119f\n"
@@ -1251,25 +1251,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
       "ldr x21, [x16, #0x28]\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
       "ldr d11, [x16, #0x30]\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "mov v10.d[1], x21\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
       "add x11, x11, #0x10\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
       "ldr x10, [x11, #0x8]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
@@ -1306,16 +1306,16 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "cmp x12, #0x8\n"
       "ldr d3, [x25, #0x0]\n"
       "add x16, x16, #0x40\n"
-      "ldr x19, [x25, #0x8]\n"
+      "ldr x8, [x25, #0x8]\n"
       "ldr d4, [x24, #0x0]\n"
       "ldr x21, [x24, #0x8]\n"
-      "mov v3.d[1], x19\n"
+      "mov v3.d[1], x8\n"
       "ldr d5, [x23, #0x0]\n"
-      "ldr x19, [x23, #0x8]\n"
+      "ldr x8, [x23, #0x8]\n"
       "mov v4.d[1], x21\n"
       "ldr d8, [x16, #0x0]\n"
       "ldr x26, [x16, #0x8]\n"
-      "mov v5.d[1], x19\n"
+      "mov v5.d[1], x8\n"
       "mov v8.d[1], x26\n"
       "bge 117b\n"
       "118:"  // Height 6: Multiply loop: Single iteration only
@@ -1382,25 +1382,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "121:"  // Height 6: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 114b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 122f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -1470,19 +1470,19 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 134f\n"
       "129:"  // Height 7: no bias
       "tbz %x[flags], #0, 133f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
       "bge 132f\n"
       "tbz x17, #1, 130f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "ldr d27, [x25], #0x8\n"
       "ldr d28, [x24], #0x8\n"
@@ -1499,7 +1499,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 131f\n"
       "130:"  // Height 7: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "ldr s27, [x25, #0x0]\n"
@@ -1507,7 +1507,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr s29, [x23, #0x0]\n"
       "ldr s30, [x22, #0x0]\n"
       "131:"  // Height 7: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 134f\n"
       "132:"  // Height 7: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -1530,11 +1530,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "135:"  // Height 7: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 136f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
@@ -1543,23 +1543,23 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr x23, [x20, #0x28]\n"
       "ldr x22, [x20, #0x30]\n"
       "cbnz x13, 137f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
+      "add x25, x25, x8, LSL #2\n"
+      "add x24, x24, x8, LSL #2\n"
+      "add x23, x23, x8, LSL #2\n"
+      "add x22, x22, x8, LSL #2\n"
       "b 137f\n"
       "136:"  // Height 7: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
-      "add x25, x27, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
+      "add x25, x27, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
       "137:"  // Height 7: input setup done
       "cmp x12, #0x4\n"
       "blt 140f\n"
@@ -1577,25 +1577,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
       "ldr x21, [x16, #0x28]\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
       "ldr d11, [x16, #0x30]\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
       "mov v10.d[1], x21\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
       "add x11, x11, #0x10\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
       "ldr x10, [x11, #0x8]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
@@ -1615,7 +1615,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v29.4s, v10.4s, v5.s[2]\n"
       "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v30.4s, v10.4s, v6.s[2]\n"
-      "ldr x19, [x25, #0x8]\n"
+      "ldr x8, [x25, #0x8]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
       "ldr d0, [x11, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
@@ -1634,7 +1634,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "add x23, x23, #0x10\n"
       "prfm pldl1keep, [x24, #0x80]\n"
       "add x22, x22, #0x10\n"
-      "mov v3.d[1], x19\n"
+      "mov v3.d[1], x8\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "sub x12, x12, #0x4\n"
@@ -1646,11 +1646,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr x26, [x16, #0x8]\n"
       "mov v4.d[1], x21\n"
       "ldr d5, [x23, #0x0]\n"
-      "ldr x19, [x23, #0x8]\n"
+      "ldr x8, [x23, #0x8]\n"
       "mov v8.d[1], x26\n"
       "ldr d6, [x22, #0x0]\n"
       "ldr x21, [x22, #0x8]\n"
-      "mov v5.d[1], x19\n"
+      "mov v5.d[1], x8\n"
       "mov v6.d[1], x21\n"
       "bge 138b\n"
       "139:"  // Height 7: Multiply loop: Single iteration only
@@ -1725,27 +1725,27 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "142:"  // Height 7: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 135b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 143f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -1809,8 +1809,8 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "mov x14, %x[output_ptr]\n"
       "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x20\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x8, #0x20\n"
+      "madd %x[output_ptr], x20, x8, %x[output_ptr]\n"
       "149:"  // Height 8: Column loop
       "cbz x15, 150f\n"
       "ldr q24, [x15, #0x0]\n"
@@ -1825,20 +1825,20 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 155f\n"
       "150:"  // Height 8: no bias
       "tbz %x[flags], #0, 154f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x17, #0x4\n"
-      "add x27, x14, x19, LSL #2\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
+      "add x21, x22, x8, LSL #2\n"
       "bge 153f\n"
       "tbz x17, #1, 151f\n"
       "ldr d24, [x14], #0x8\n"
       "ldr d25, [x27], #0x8\n"
-      "mov x19, #0x8\n"
+      "mov x8, #0x8\n"
       "ldr d26, [x26], #0x8\n"
       "ldr d27, [x25], #0x8\n"
       "ldr d28, [x24], #0x8\n"
@@ -1857,7 +1857,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "b 152f\n"
       "151:"  // Height 8: Partial accumulate: partial_1_0
       "ldr s24, [x14, #0x0]\n"
-      "mov x19, #0x0\n"
+      "mov x8, #0x0\n"
       "ldr s25, [x27, #0x0]\n"
       "ldr s26, [x26, #0x0]\n"
       "ldr s27, [x25, #0x0]\n"
@@ -1866,7 +1866,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr s30, [x22, #0x0]\n"
       "ldr s31, [x21, #0x0]\n"
       "152:"  // Height 8: Partial accumulate: Done
-      "sub x14, x14, x19\n"
+      "sub x14, x14, x8\n"
       "b 155f\n"
       "153:"  // Height 8: full accumulate
       "ldr q24, [x14, #0x0]\n"
@@ -1891,11 +1891,11 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "mov x13, #0x0\n"
       "156:"  // Height 8: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w12, [x20, x13, LSL #0x2]\n"
       "tbz %x[flags], #3, 157f\n"
       "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
+      "add x20, x20, x8, LSL #3\n"
       "ldr x11, [x20, #0x0]\n"
       "ldr x9, [x20, #0x8]\n"
       "ldr x27, [x20, #0x10]\n"
@@ -1905,25 +1905,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr x22, [x20, #0x30]\n"
       "ldr x20, [x20, #0x38]\n"
       "cbnz x13, 158f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x11, x11, x19, LSL #2\n"
-      "add x9, x9, x19, LSL #2\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x11, x11, x8, LSL #2\n"
+      "add x9, x9, x8, LSL #2\n"
+      "add x27, x27, x8, LSL #2\n"
+      "add x25, x25, x8, LSL #2\n"
+      "add x24, x24, x8, LSL #2\n"
+      "add x23, x23, x8, LSL #2\n"
+      "add x22, x22, x8, LSL #2\n"
+      "add x20, x20, x8, LSL #2\n"
       "b 158f\n"
       "157:"  // Height 8: setup direct input
       "mov x11, %x[input_ptr]\n"
-      "add x9, x11, x19, LSL #2\n"
-      "add x27, x9, x19, LSL #2\n"
-      "add x25, x27, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x20, x22, x19, LSL #2\n"
+      "add x9, x11, x8, LSL #2\n"
+      "add x27, x9, x8, LSL #2\n"
+      "add x25, x27, x8, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
+      "add x20, x22, x8, LSL #2\n"
       "158:"  // Height 8: input setup done
       "cmp x12, #0x4\n"
       "blt 161f\n"
@@ -1942,25 +1942,25 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v24.4s, v8.4s, v0.s[0]\n"
       "ldr d9, [x16, #0x10]\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr x19, [x16, #0x18]\n"
+      "ldr x8, [x16, #0x18]\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
       "ldr d10, [x16, #0x20]\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
       "ldr x21, [x16, #0x28]\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v9.d[1], x8\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
       "ldr d11, [x16, #0x30]\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
       "mov v10.d[1], x21\n"
       "fmla v31.4s, v8.4s, v7.s[0]\n"
-      "ldr x19, [x16, #0x38]\n"
+      "ldr x8, [x16, #0x38]\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
       "add x11, x11, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
       "prfm pldl1keep, [x11, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "mov v11.d[1], x19\n"
+      "mov v11.d[1], x8\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "ldr x10, [x11, #0x8]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
@@ -1980,7 +1980,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
-      "ldr x19, [x25, #0x8]\n"
+      "ldr x8, [x25, #0x8]\n"
       "fmla v29.4s, v10.4s, v5.s[2]\n"
       "add x24, x24, #0x10\n"
       "fmla v30.4s, v10.4s, v6.s[2]\n"
@@ -2005,7 +2005,7 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "ldr d4, [x24, #0x0]\n"
       "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "mov v3.d[1], x19\n"
+      "mov v3.d[1], x8\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "mov v4.d[1], x21\n"
       "prfm pldl1keep, [x22, #0x80]\n"
@@ -2013,19 +2013,19 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "add x20, x20, #0x10\n"
       "prfm pldl1keep, [x20, #0x80]\n"
       "sub x12, x12, #0x4\n"
-      "ldr x19, [x23, #0x8]\n"
+      "ldr x8, [x23, #0x8]\n"
       "cmp x12, #0x8\n"
       "ldr d6, [x22, #0x0]\n"
       "add x16, x16, #0x40\n"
       "ldr d8, [x16, #0x0]\n"
-      "mov v5.d[1], x19\n"
+      "mov v5.d[1], x8\n"
       "ldr x26, [x16, #0x8]\n"
       "ldr x21, [x22, #0x8]\n"
       "ldr d7, [x20, #0x0]\n"
       "mov v8.d[1], x26\n"
-      "ldr x19, [x20, #0x8]\n"
+      "ldr x8, [x20, #0x8]\n"
       "mov v6.d[1], x21\n"
-      "mov v7.d[1], x19\n"
+      "mov v7.d[1], x8\n"
       "bge 159b\n"
       "160:"  // Height 8: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
@@ -2107,29 +2107,29 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "163:"  // Height 8: Multiply loop: No odd multiplies
       "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x13, x13, #0x1\n"
-      "cmp x13, x19\n"
+      "cmp x13, x8\n"
       "bne 156b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "prfm pstl1keep, [x14, #0x0]\n"
-      "add x27, x14, x19, LSL #2\n"
+      "add x27, x14, x8, LSL #2\n"
       "prfm pstl1keep, [x27, #0x0]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "add x26, x27, x8, LSL #2\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x25, x26, x8, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x24, x25, x8, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x23, x24, x8, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x22, x23, x8, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x21, x22, x8, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 164f\n"
       "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x8, %x[args_ptr], %[offset_max]\n"
       "ld1r { v17.4s }, [x20]\n"
-      "ld1r { v16.4s }, [x19]\n"
+      "ld1r { v16.4s }, [x8]\n"
       "fmin v24.4s, v24.4s, v16.4s\n"
       "fmin v25.4s, v25.4s, v16.4s\n"
       "fmin v26.4s, v26.4s, v16.4s\n"
@@ -2200,14 +2200,14 @@ void a64_hybrid_fp32_mla_8x4_a55 (
       "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "169:"  // Update direct input
-      "mov x19, #0x20\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x8, #0x20\n"
+      "madd %x[input_ptr], x8, x20, %x[input_ptr]\n"
       "b 1b\n"
       "170:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp
index 9bed0213da..bd22336c8d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -105,527 +105,527 @@ void a64_hybrid_fp32_mla_8x4 (
       "cmp %x[M], #0x2\n"
       "bgt 43f\n"
       "beq 22f\n"
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x11, %x[bias]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x11, 3f\n"
-      "ldr q24, [x11, #0x0]\n"
-      "add x11, x11, #0x10\n"
+      "cbz x14, 3f\n"
+      "ldr q24, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
       "b 8f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 7f\n"
       "cmp x13, #0x4\n"
       "bge 6f\n"
       "tbz x13, #1, 4f\n"
-      "ldr d24, [x10], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x13, #0, 5f\n"
-      "ld1 { v24.s }[2], [x10]\n"
+      "ld1 { v24.s }[2], [x11]\n"
       "b 5f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s24, [x11, #0x0]\n"
+      "mov x20, #0x0\n"
       "5:"  // Height 1: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 8f\n"
       "6:"  // Height 1: full accumulate
-      "ldr q24, [x10, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
       "b 8f\n"
       "7:"  // Height 1: no accumulate
       "movi v24.16b, #0x0\n"
       "8:"  // Height 1: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "9:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 10f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "cbnz x9, 11f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "cbnz x10, 11f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
       "b 11f\n"
       "10:"  // Height 1: setup direct input
-      "mov x27, %x[input_ptr]\n"
+      "mov x28, %x[input_ptr]\n"
       "11:"  // Height 1: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 14f\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
-      "cmp x28, #0x8\n"
+      "cmp x9, #0x8\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 13f\n"
       "12:"  // Height 1: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "sub x28, x28, #0x4\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "cmp x28, #0x8\n"
-      "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "cmp x9, #0x8\n"
       "add x12, x12, #0x40\n"
-      "ldr q0, [x27, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "fmla v24.4s, v11.4s, v0.s[3]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "bge 12b\n"
       "13:"  // Height 1: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x27, x27, #0x10\n"
+      "add x28, x28, #0x10\n"
+      "sub x9, x9, #0x4\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "14:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x28, 16f\n"
+      "cbz x9, 16f\n"
       "15:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
+      "ldr s0, [x28], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
+      "sub x9, x9, #0x1\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
       "add x12, x12, #0x10\n"
-      "cbnz x28, 15b\n"
+      "cbnz x9, 15b\n"
       "16:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 9b\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
       "tbz %x[flags], #1, 17f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
       "17:"  // Height 1: No activation
       "cmp x13, #0x4\n"
       "bge 20f\n"
       "tbz x13, #1, 18f\n"
-      "str d24, [x10], #0x8\n"
+      "str d24, [x11], #0x8\n"
       "tbz x13, #0, 19f\n"
-      "st1 { v24.s }[2], [x10]\n"
+      "st1 { v24.s }[2], [x11]\n"
       "b 19f\n"
       "18:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
       "19:"  // Height 1: Partial direct writeback: Done
       "b 21f\n"
       "20:"  // Height 1: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
       "21:"  // Height 1: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 2b\n"
       "b 170f\n"
       "22:"  // Height 2
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "23:"  // Height 2: Column loop
-      "cbz x11, 24f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 24f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
+      "add x14, x14, #0x10\n"
       "b 29f\n"
       "24:"  // Height 2: no bias
       "tbz %x[flags], #0, 28f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
+      "add x27, x11, x20, LSL #2\n"
       "bge 27f\n"
       "tbz x13, #1, 25f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x13, #0, 26f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
       "b 26f\n"
       "25:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
       "26:"  // Height 2: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 29f\n"
       "27:"  // Height 2: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
       "b 29f\n"
       "28:"  // Height 2: no accumulate
       "movi v24.16b, #0x0\n"
       "movi v25.16b, #0x0\n"
       "29:"  // Height 2: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "30:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "cbnz x9, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "cbnz x10, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
       "b 32f\n"
       "31:"  // Height 2: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
       "32:"  // Height 2: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 35f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 34f\n"
       "33:"  // Height 2: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "cmp x28, #0x8\n"
+      "add x27, x27, #0x10\n"
+      "cmp x9, #0x8\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "add x12, x12, #0x40\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "bge 33b\n"
       "34:"  // Height 2: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x12, x12, #0x40\n"
+      "sub x9, x9, #0x4\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
       "35:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x28, 37f\n"
+      "cbz x9, 37f\n"
       "36:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
-      "cbnz x28, 36b\n"
+      "add x12, x12, #0x10\n"
+      "cbnz x9, 36b\n"
       "37:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
       "38:"  // Height 2: No activation
       "cmp x13, #0x4\n"
       "bge 41f\n"
       "tbz x13, #1, 39f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
       "tbz x13, #0, 40f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
       "b 40f\n"
       "39:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
       "40:"  // Height 2: Partial direct writeback: Done
       "b 42f\n"
       "41:"  // Height 2: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
       "42:"  // Height 2: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 23b\n"
       "b 170f\n"
       "43:"  // Height 3
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "44:"  // Height 3: Column loop
-      "cbz x11, 45f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 45f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "b 50f\n"
       "45:"  // Height 3: no bias
       "tbz %x[flags], #0, 49f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "bge 48f\n"
       "tbz x13, #1, 46f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
       "tbz x13, #0, 47f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
       "b 47f\n"
       "46:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
       "47:"  // Height 3: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 50f\n"
       "48:"  // Height 3: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
       "b 50f\n"
       "49:"  // Height 3: no accumulate
       "movi v24.16b, #0x0\n"
       "movi v25.16b, #0x0\n"
       "movi v26.16b, #0x0\n"
       "50:"  // Height 3: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "51:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 52f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "cbnz x9, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "cbnz x10, 53f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 53f\n"
       "52:"  // Height 3: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "53:"  // Height 3: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 56f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 55f\n"
       "54:"  // Height 3: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "sub x28, x28, #0x4\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "cmp x28, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "cmp x9, #0x8\n"
       "add x12, x12, #0x40\n"
-      "fmla v24.4s, v10.4s, v0.s[2]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 54b\n"
       "55:"  // Height 3: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
       "56:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x28, 58f\n"
+      "cbz x9, 58f\n"
       "57:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
-      "cbnz x28, 57b\n"
+      "add x12, x12, #0x10\n"
+      "cbnz x9, 57b\n"
       "58:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 51b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "prfm pstl1keep, [x25, #0x0]\n"
       "tbz %x[flags], #1, 59f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
       "59:"  // Height 3: No activation
       "cmp x13, #0x4\n"
       "bge 62f\n"
       "tbz x13, #1, 60f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
       "tbz x13, #0, 61f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
       "b 61f\n"
       "60:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
       "61:"  // Height 3: Partial direct writeback: Done
       "b 63f\n"
       "62:"  // Height 3: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
       "63:"  // Height 3: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 44b\n"
       "b 170f\n"
       "64:"  // Height 4
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "65:"  // Height 4: Column loop
-      "cbz x11, 66f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 66f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "mov v27.16b, v24.16b\n"
       "b 71f\n"
       "66:"  // Height 4: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "bge 69f\n"
       "tbz x13, #1, 67f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
-      "ldr d27, [x24], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
+      "ldr d27, [x25], #0x8\n"
       "tbz x13, #0, 68f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
-      "ld1 { v27.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
+      "ld1 { v27.s }[2], [x25]\n"
       "b 68f\n"
       "67:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
-      "ldr s27, [x24, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
+      "ldr s27, [x25, #0x0]\n"
       "68:"  // Height 4: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 71f\n"
       "69:"  // Height 4: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
-      "ldr q27, [x24, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
+      "ldr q27, [x25, #0x0]\n"
       "b 71f\n"
       "70:"  // Height 4: no accumulate
       "movi v24.16b, #0x0\n"
@@ -633,101 +633,101 @@ void a64_hybrid_fp32_mla_8x4 (
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
       "71:"  // Height 4: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "72:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x9, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x10, 74f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 74f\n"
       "73:"  // Height 4: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "74:"  // Height 4: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 77f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 76f\n"
       "75:"  // Height 4: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x28, #0x8\n"
+      "add x25, x25, #0x10\n"
+      "cmp x9, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "add x12, x12, #0x40\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "bge 75b\n"
       "76:"  // Height 4: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x12, x12, #0x40\n"
+      "sub x9, x9, #0x4\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
@@ -735,130 +735,130 @@ void a64_hybrid_fp32_mla_8x4 (
       "fmla v26.4s, v11.4s, v2.s[3]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
       "77:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x28, 79f\n"
+      "cbz x9, 79f\n"
       "78:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
       "fmla v27.4s, v12.4s, v3.s[0]\n"
-      "cbnz x28, 78b\n"
+      "cbnz x9, 78b\n"
       "79:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
       "tbz %x[flags], #1, 80f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmin v27.4s, v27.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
-      "fmax v27.4s, v27.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmin v27.4s, v27.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
+      "fmax v27.4s, v27.4s, v16.4s\n"
       "80:"  // Height 4: No activation
       "cmp x13, #0x4\n"
       "bge 83f\n"
       "tbz x13, #1, 81f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
-      "str d27, [x24], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
+      "str d27, [x25], #0x8\n"
       "tbz x13, #0, 82f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
-      "st1 { v27.s }[2], [x24]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
+      "st1 { v27.s }[2], [x25]\n"
       "b 82f\n"
       "81:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
-      "str s27, [x24, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
+      "str s27, [x25, #0x0]\n"
       "82:"  // Height 4: Partial direct writeback: Done
       "b 84f\n"
       "83:"  // Height 4: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
-      "str q27, [x24, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
+      "str q27, [x25, #0x0]\n"
       "84:"  // Height 4: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 65b\n"
       "b 170f\n"
       "85:"  // Height 5
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "86:"  // Height 5: Column loop
-      "cbz x11, 87f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 87f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "mov v27.16b, v24.16b\n"
       "mov v28.16b, v24.16b\n"
       "b 92f\n"
       "87:"  // Height 5: no bias
       "tbz %x[flags], #0, 91f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 90f\n"
       "tbz x13, #1, 88f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
-      "ldr d27, [x24], #0x8\n"
-      "ldr d28, [x23], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
+      "ldr d27, [x25], #0x8\n"
+      "ldr d28, [x24], #0x8\n"
       "tbz x13, #0, 89f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
-      "ld1 { v27.s }[2], [x24]\n"
-      "ld1 { v28.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
+      "ld1 { v27.s }[2], [x25]\n"
+      "ld1 { v28.s }[2], [x24]\n"
       "b 89f\n"
       "88:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
-      "ldr s27, [x24, #0x0]\n"
-      "ldr s28, [x23, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
+      "ldr s27, [x25, #0x0]\n"
+      "ldr s28, [x24, #0x0]\n"
       "89:"  // Height 5: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 92f\n"
       "90:"  // Height 5: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
-      "ldr q27, [x24, #0x0]\n"
-      "ldr q28, [x23, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
+      "ldr q27, [x25, #0x0]\n"
+      "ldr q28, [x24, #0x0]\n"
       "b 92f\n"
       "91:"  // Height 5: no accumulate
       "movi v24.16b, #0x0\n"
@@ -867,116 +867,116 @@ void a64_hybrid_fp32_mla_8x4 (
       "movi v27.16b, #0x0\n"
       "movi v28.16b, #0x0\n"
       "92:"  // Height 5: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "93:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 94f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "cbnz x9, 95f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "cbnz x10, 95f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 95f\n"
       "94:"  // Height 5: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "95:"  // Height 5: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 98f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x23, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 97f\n"
       "96:"  // Height 5: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x28, x28, #0x4\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "cmp x28, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "cmp x9, #0x8\n"
       "add x12, x12, #0x40\n"
-      "fmla v27.4s, v9.4s, v3.s[1]\n"
       "ldr q8, [x12, #0x0]\n"
+      "fmla v27.4s, v9.4s, v3.s[1]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
+      "ldr q9, [x12, #0x10]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "fmla v28.4s, v11.4s, v4.s[3]\n"
-      "ldr q4, [x23, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
       "bge 96b\n"
       "97:"  // Height 5: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "sub x9, x9, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
@@ -986,146 +986,146 @@ void a64_hybrid_fp32_mla_8x4 (
       "fmla v27.4s, v11.4s, v3.s[3]\n"
       "fmla v28.4s, v11.4s, v4.s[3]\n"
       "98:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x28, 100f\n"
+      "cbz x9, 100f\n"
       "99:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x23], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x24], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
       "fmla v27.4s, v12.4s, v3.s[0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v28.4s, v12.4s, v4.s[0]\n"
-      "cbnz x28, 99b\n"
+      "cbnz x9, 99b\n"
       "100:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 93b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "tbz %x[flags], #1, 101f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmin v27.4s, v27.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
-      "fmax v27.4s, v27.4s, v17.4s\n"
-      "fmin v28.4s, v28.4s, v16.4s\n"
-      "fmax v28.4s, v28.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmin v27.4s, v27.4s, v17.4s\n"
+      "fmin v28.4s, v28.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
+      "fmax v27.4s, v27.4s, v16.4s\n"
+      "fmax v28.4s, v28.4s, v16.4s\n"
       "101:"  // Height 5: No activation
       "cmp x13, #0x4\n"
       "bge 104f\n"
       "tbz x13, #1, 102f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
-      "str d27, [x24], #0x8\n"
-      "str d28, [x23], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
+      "str d27, [x25], #0x8\n"
+      "str d28, [x24], #0x8\n"
       "tbz x13, #0, 103f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
-      "st1 { v27.s }[2], [x24]\n"
-      "st1 { v28.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
+      "st1 { v27.s }[2], [x25]\n"
+      "st1 { v28.s }[2], [x24]\n"
       "b 103f\n"
       "102:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
-      "str s27, [x24, #0x0]\n"
-      "str s28, [x23, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
+      "str s27, [x25, #0x0]\n"
+      "str s28, [x24, #0x0]\n"
       "103:"  // Height 5: Partial direct writeback: Done
       "b 105f\n"
       "104:"  // Height 5: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
-      "str q27, [x24, #0x0]\n"
-      "str q28, [x23, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
+      "str q27, [x25, #0x0]\n"
+      "str q28, [x24, #0x0]\n"
       "105:"  // Height 5: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 86b\n"
       "b 170f\n"
       "106:"  // Height 6
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "107:"  // Height 6: Column loop
-      "cbz x11, 108f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 108f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "mov v27.16b, v24.16b\n"
       "mov v28.16b, v24.16b\n"
       "mov v29.16b, v24.16b\n"
       "b 113f\n"
       "108:"  // Height 6: no bias
       "tbz %x[flags], #0, 112f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 111f\n"
       "tbz x13, #1, 109f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
-      "ldr d27, [x24], #0x8\n"
-      "ldr d28, [x23], #0x8\n"
-      "ldr d29, [x22], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
+      "ldr d27, [x25], #0x8\n"
+      "ldr d28, [x24], #0x8\n"
+      "ldr d29, [x23], #0x8\n"
       "tbz x13, #0, 110f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
-      "ld1 { v27.s }[2], [x24]\n"
-      "ld1 { v28.s }[2], [x23]\n"
-      "ld1 { v29.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
+      "ld1 { v27.s }[2], [x25]\n"
+      "ld1 { v28.s }[2], [x24]\n"
+      "ld1 { v29.s }[2], [x23]\n"
       "b 110f\n"
       "109:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
-      "ldr s27, [x24, #0x0]\n"
-      "ldr s28, [x23, #0x0]\n"
-      "ldr s29, [x22, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
+      "ldr s27, [x25, #0x0]\n"
+      "ldr s28, [x24, #0x0]\n"
+      "ldr s29, [x23, #0x0]\n"
       "110:"  // Height 6: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 113f\n"
       "111:"  // Height 6: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
-      "ldr q27, [x24, #0x0]\n"
-      "ldr q28, [x23, #0x0]\n"
-      "ldr q29, [x22, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
+      "ldr q27, [x25, #0x0]\n"
+      "ldr q28, [x24, #0x0]\n"
+      "ldr q29, [x23, #0x0]\n"
       "b 113f\n"
       "112:"  // Height 6: no accumulate
       "movi v24.16b, #0x0\n"
@@ -1135,131 +1135,131 @@ void a64_hybrid_fp32_mla_8x4 (
       "movi v28.16b, #0x0\n"
       "movi v29.16b, #0x0\n"
       "113:"  // Height 6: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "114:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 115f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "cbnz x9, 116f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "cbnz x10, 116f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 116f\n"
       "115:"  // Height 6: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "116:"  // Height 6: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 119f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x23, #0x0]\n"
-      "ldr q5, [x22, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 118f\n"
       "117:"  // Height 6: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x28, #0x8\n"
+      "add x23, x23, #0x10\n"
+      "cmp x9, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "add x12, x12, #0x40\n"
       "ldr q8, [x12, #0x0]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
       "fmla v29.4s, v10.4s, v5.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "fmla v28.4s, v11.4s, v4.s[3]\n"
-      "ldr q4, [x23, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
       "fmla v29.4s, v11.4s, v5.s[3]\n"
-      "ldr q5, [x22, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
       "bge 117b\n"
       "118:"  // Height 6: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x22, x22, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x12, x12, #0x40\n"
+      "sub x9, x9, #0x4\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
@@ -1271,108 +1271,108 @@ void a64_hybrid_fp32_mla_8x4 (
       "fmla v28.4s, v11.4s, v4.s[3]\n"
       "fmla v29.4s, v11.4s, v5.s[3]\n"
       "119:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x28, 121f\n"
+      "cbz x9, 121f\n"
       "120:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x23], #0x4\n"
-      "ldr s5, [x22], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x24], #0x4\n"
+      "ldr s5, [x23], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
       "fmla v27.4s, v12.4s, v3.s[0]\n"
       "fmla v28.4s, v12.4s, v4.s[0]\n"
       "fmla v29.4s, v12.4s, v5.s[0]\n"
-      "cbnz x28, 120b\n"
+      "cbnz x9, 120b\n"
       "121:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 114b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "tbz %x[flags], #1, 122f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmin v27.4s, v27.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
-      "fmax v27.4s, v27.4s, v17.4s\n"
-      "fmin v28.4s, v28.4s, v16.4s\n"
-      "fmin v29.4s, v29.4s, v16.4s\n"
-      "fmax v28.4s, v28.4s, v17.4s\n"
-      "fmax v29.4s, v29.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmin v27.4s, v27.4s, v17.4s\n"
+      "fmin v28.4s, v28.4s, v17.4s\n"
+      "fmin v29.4s, v29.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
+      "fmax v27.4s, v27.4s, v16.4s\n"
+      "fmax v28.4s, v28.4s, v16.4s\n"
+      "fmax v29.4s, v29.4s, v16.4s\n"
       "122:"  // Height 6: No activation
       "cmp x13, #0x4\n"
       "bge 125f\n"
       "tbz x13, #1, 123f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
-      "str d27, [x24], #0x8\n"
-      "str d28, [x23], #0x8\n"
-      "str d29, [x22], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
+      "str d27, [x25], #0x8\n"
+      "str d28, [x24], #0x8\n"
+      "str d29, [x23], #0x8\n"
       "tbz x13, #0, 124f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
-      "st1 { v27.s }[2], [x24]\n"
-      "st1 { v28.s }[2], [x23]\n"
-      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
+      "st1 { v27.s }[2], [x25]\n"
+      "st1 { v28.s }[2], [x24]\n"
+      "st1 { v29.s }[2], [x23]\n"
       "b 124f\n"
       "123:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
-      "str s27, [x24, #0x0]\n"
-      "str s28, [x23, #0x0]\n"
-      "str s29, [x22, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
+      "str s27, [x25, #0x0]\n"
+      "str s28, [x24, #0x0]\n"
+      "str s29, [x23, #0x0]\n"
       "124:"  // Height 6: Partial direct writeback: Done
       "b 126f\n"
       "125:"  // Height 6: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
-      "str q27, [x24, #0x0]\n"
-      "str q28, [x23, #0x0]\n"
-      "str q29, [x22, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
+      "str q27, [x25, #0x0]\n"
+      "str q28, [x24, #0x0]\n"
+      "str q29, [x23, #0x0]\n"
       "126:"  // Height 6: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 107b\n"
       "b 170f\n"
       "127:"  // Height 7
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "128:"  // Height 7: Column loop
-      "cbz x11, 129f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 129f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "mov v27.16b, v24.16b\n"
       "mov v28.16b, v24.16b\n"
       "mov v29.16b, v24.16b\n"
@@ -1380,53 +1380,53 @@ void a64_hybrid_fp32_mla_8x4 (
       "b 134f\n"
       "129:"  // Height 7: no bias
       "tbz %x[flags], #0, 133f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 132f\n"
       "tbz x13, #1, 130f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
-      "ldr d27, [x24], #0x8\n"
-      "ldr d28, [x23], #0x8\n"
-      "ldr d29, [x22], #0x8\n"
-      "ldr d30, [x21], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
+      "ldr d27, [x25], #0x8\n"
+      "ldr d28, [x24], #0x8\n"
+      "ldr d29, [x23], #0x8\n"
+      "ldr d30, [x22], #0x8\n"
       "tbz x13, #0, 131f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
-      "ld1 { v27.s }[2], [x24]\n"
-      "ld1 { v28.s }[2], [x23]\n"
-      "ld1 { v29.s }[2], [x22]\n"
-      "ld1 { v30.s }[2], [x21]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
+      "ld1 { v27.s }[2], [x25]\n"
+      "ld1 { v28.s }[2], [x24]\n"
+      "ld1 { v29.s }[2], [x23]\n"
+      "ld1 { v30.s }[2], [x22]\n"
       "b 131f\n"
       "130:"  // Height 7: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
-      "ldr s27, [x24, #0x0]\n"
-      "ldr s28, [x23, #0x0]\n"
-      "ldr s29, [x22, #0x0]\n"
-      "ldr s30, [x21, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
+      "ldr s27, [x25, #0x0]\n"
+      "ldr s28, [x24, #0x0]\n"
+      "ldr s29, [x23, #0x0]\n"
+      "ldr s30, [x22, #0x0]\n"
       "131:"  // Height 7: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 134f\n"
       "132:"  // Height 7: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
-      "ldr q27, [x24, #0x0]\n"
-      "ldr q28, [x23, #0x0]\n"
-      "ldr q29, [x22, #0x0]\n"
-      "ldr q30, [x21, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
+      "ldr q27, [x25, #0x0]\n"
+      "ldr q28, [x24, #0x0]\n"
+      "ldr q29, [x23, #0x0]\n"
+      "ldr q30, [x22, #0x0]\n"
       "b 134f\n"
       "133:"  // Height 7: no accumulate
       "movi v24.16b, #0x0\n"
@@ -1437,146 +1437,146 @@ void a64_hybrid_fp32_mla_8x4 (
       "movi v29.16b, #0x0\n"
       "movi v30.16b, #0x0\n"
       "134:"  // Height 7: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "135:"  // Height 7: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 136f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "cbnz x9, 137f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "cbnz x10, 137f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 137f\n"
       "136:"  // Height 7: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "137:"  // Height 7: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 140f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x23, #0x0]\n"
-      "ldr q5, [x22, #0x0]\n"
-      "ldr q6, [x21, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 139f\n"
       "138:"  // Height 7: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x28, x28, #0x4\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x28, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "cmp x9, #0x8\n"
       "add x12, x12, #0x40\n"
-      "fmla v27.4s, v9.4s, v3.s[1]\n"
       "ldr q8, [x12, #0x0]\n"
+      "fmla v27.4s, v9.4s, v3.s[1]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
       "fmla v30.4s, v9.4s, v6.s[1]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
       "fmla v29.4s, v10.4s, v5.s[2]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v30.4s, v10.4s, v6.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "fmla v28.4s, v11.4s, v4.s[3]\n"
-      "ldr q4, [x23, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
       "fmla v29.4s, v11.4s, v5.s[3]\n"
-      "ldr q5, [x22, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
       "fmla v30.4s, v11.4s, v6.s[3]\n"
-      "ldr q6, [x21, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
       "bge 138b\n"
       "139:"  // Height 7: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x22, x22, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
+      "sub x9, x9, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
       "fmla v30.4s, v9.4s, v6.s[1]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
@@ -1590,121 +1590,121 @@ void a64_hybrid_fp32_mla_8x4 (
       "fmla v29.4s, v11.4s, v5.s[3]\n"
       "fmla v30.4s, v11.4s, v6.s[3]\n"
       "140:"  // Height 7: Multiply loop: Main loop skip
-      "cbz x28, 142f\n"
+      "cbz x9, 142f\n"
       "141:"  // Height 7: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x23], #0x4\n"
-      "ldr s5, [x22], #0x4\n"
-      "ldr s6, [x21], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x24], #0x4\n"
+      "ldr s5, [x23], #0x4\n"
+      "ldr s6, [x22], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
       "fmla v27.4s, v12.4s, v3.s[0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v28.4s, v12.4s, v4.s[0]\n"
       "fmla v29.4s, v12.4s, v5.s[0]\n"
       "fmla v30.4s, v12.4s, v6.s[0]\n"
-      "cbnz x28, 141b\n"
+      "cbnz x9, 141b\n"
       "142:"  // Height 7: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 135b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "tbz %x[flags], #1, 143f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmin v27.4s, v27.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
-      "fmax v27.4s, v27.4s, v17.4s\n"
-      "fmin v28.4s, v28.4s, v16.4s\n"
-      "fmin v29.4s, v29.4s, v16.4s\n"
-      "fmin v30.4s, v30.4s, v16.4s\n"
-      "fmax v28.4s, v28.4s, v17.4s\n"
-      "fmax v29.4s, v29.4s, v17.4s\n"
-      "fmax v30.4s, v30.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmin v27.4s, v27.4s, v17.4s\n"
+      "fmin v28.4s, v28.4s, v17.4s\n"
+      "fmin v29.4s, v29.4s, v17.4s\n"
+      "fmin v30.4s, v30.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
+      "fmax v27.4s, v27.4s, v16.4s\n"
+      "fmax v28.4s, v28.4s, v16.4s\n"
+      "fmax v29.4s, v29.4s, v16.4s\n"
+      "fmax v30.4s, v30.4s, v16.4s\n"
       "143:"  // Height 7: No activation
       "cmp x13, #0x4\n"
       "bge 146f\n"
       "tbz x13, #1, 144f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
-      "str d27, [x24], #0x8\n"
-      "str d28, [x23], #0x8\n"
-      "str d29, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
+      "str d27, [x25], #0x8\n"
+      "str d28, [x24], #0x8\n"
+      "str d29, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
       "tbz x13, #0, 145f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
-      "st1 { v27.s }[2], [x24]\n"
-      "st1 { v28.s }[2], [x23]\n"
-      "st1 { v29.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
+      "st1 { v27.s }[2], [x25]\n"
+      "st1 { v28.s }[2], [x24]\n"
+      "st1 { v29.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
       "b 145f\n"
       "144:"  // Height 7: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
-      "str s27, [x24, #0x0]\n"
-      "str s28, [x23, #0x0]\n"
-      "str s29, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
+      "str s27, [x25, #0x0]\n"
+      "str s28, [x24, #0x0]\n"
+      "str s29, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
       "145:"  // Height 7: Partial direct writeback: Done
       "b 147f\n"
       "146:"  // Height 7: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
-      "str q27, [x24, #0x0]\n"
-      "str q28, [x23, #0x0]\n"
-      "str q29, [x22, #0x0]\n"
-      "str q30, [x21, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
+      "str q27, [x25, #0x0]\n"
+      "str q28, [x24, #0x0]\n"
+      "str q29, [x23, #0x0]\n"
+      "str q30, [x22, #0x0]\n"
       "147:"  // Height 7: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 128b\n"
       "b 170f\n"
       "148:"  // Height 8
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x20\n"
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x20\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "149:"  // Height 8: Column loop
-      "cbz x11, 150f\n"
-      "ldr q24, [x11, #0x0]\n"
+      "cbz x14, 150f\n"
+      "ldr q24, [x14, #0x0]\n"
       "mov v25.16b, v24.16b\n"
-      "add x11, x11, #0x10\n"
       "mov v26.16b, v24.16b\n"
+      "add x14, x14, #0x10\n"
       "mov v27.16b, v24.16b\n"
       "mov v28.16b, v24.16b\n"
       "mov v29.16b, v24.16b\n"
@@ -1713,58 +1713,58 @@ void a64_hybrid_fp32_mla_8x4 (
       "b 155f\n"
       "150:"  // Height 8: no bias
       "tbz %x[flags], #0, 154f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x13, #0x4\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 153f\n"
       "tbz x13, #1, 151f\n"
-      "ldr d24, [x10], #0x8\n"
-      "ldr d25, [x26], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d26, [x25], #0x8\n"
-      "ldr d27, [x24], #0x8\n"
-      "ldr d28, [x23], #0x8\n"
-      "ldr d29, [x22], #0x8\n"
-      "ldr d30, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d24, [x11], #0x8\n"
+      "ldr d25, [x27], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d26, [x26], #0x8\n"
+      "ldr d27, [x25], #0x8\n"
+      "ldr d28, [x24], #0x8\n"
+      "ldr d29, [x23], #0x8\n"
+      "ldr d30, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x13, #0, 152f\n"
-      "ld1 { v24.s }[2], [x10]\n"
-      "ld1 { v25.s }[2], [x26]\n"
-      "ld1 { v26.s }[2], [x25]\n"
-      "ld1 { v27.s }[2], [x24]\n"
-      "ld1 { v28.s }[2], [x23]\n"
-      "ld1 { v29.s }[2], [x22]\n"
-      "ld1 { v30.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v24.s }[2], [x11]\n"
+      "ld1 { v25.s }[2], [x27]\n"
+      "ld1 { v26.s }[2], [x26]\n"
+      "ld1 { v27.s }[2], [x25]\n"
+      "ld1 { v28.s }[2], [x24]\n"
+      "ld1 { v29.s }[2], [x23]\n"
+      "ld1 { v30.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 152f\n"
       "151:"  // Height 8: Partial accumulate: partial_1_0
-      "ldr s24, [x10, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s25, [x26, #0x0]\n"
-      "ldr s26, [x25, #0x0]\n"
-      "ldr s27, [x24, #0x0]\n"
-      "ldr s28, [x23, #0x0]\n"
-      "ldr s29, [x22, #0x0]\n"
-      "ldr s30, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s24, [x11, #0x0]\n"
+      "ldr s25, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s26, [x26, #0x0]\n"
+      "ldr s27, [x25, #0x0]\n"
+      "ldr s28, [x24, #0x0]\n"
+      "ldr s29, [x23, #0x0]\n"
+      "ldr s30, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "152:"  // Height 8: Partial accumulate: Done
-      "sub x10, x10, x19\n"
+      "sub x11, x11, x20\n"
       "b 155f\n"
       "153:"  // Height 8: full accumulate
-      "ldr q24, [x10, #0x0]\n"
-      "ldr q25, [x26, #0x0]\n"
-      "ldr q26, [x25, #0x0]\n"
-      "ldr q27, [x24, #0x0]\n"
-      "ldr q28, [x23, #0x0]\n"
-      "ldr q29, [x22, #0x0]\n"
-      "ldr q30, [x21, #0x0]\n"
-      "ldr q31, [x20, #0x0]\n"
+      "ldr q24, [x11, #0x0]\n"
+      "ldr q25, [x27, #0x0]\n"
+      "ldr q26, [x26, #0x0]\n"
+      "ldr q27, [x25, #0x0]\n"
+      "ldr q28, [x24, #0x0]\n"
+      "ldr q29, [x23, #0x0]\n"
+      "ldr q30, [x22, #0x0]\n"
+      "ldr q31, [x21, #0x0]\n"
       "b 155f\n"
       "154:"  // Height 8: no accumulate
       "movi v24.16b, #0x0\n"
@@ -1776,161 +1776,161 @@ void a64_hybrid_fp32_mla_8x4 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "155:"  // Height 8: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "156:"  // Height 8: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 157f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "ldr x20, [x20, #0x38]\n"
-      "cbnz x9, 158f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "ldr x21, [x21, #0x38]\n"
+      "cbnz x10, 158f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 158f\n"
       "157:"  // Height 8: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "158:"  // Height 8: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "blt 161f\n"
-      "ldr q0, [x27, #0x0]\n"
-      "ldr q1, [x26, #0x0]\n"
-      "cmp x28, #0x8\n"
-      "ldr q2, [x25, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x23, #0x0]\n"
-      "ldr q5, [x22, #0x0]\n"
-      "ldr q6, [x21, #0x0]\n"
-      "ldr q7, [x20, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
+      "cmp x9, #0x8\n"
+      "ldr q2, [x26, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
+      "ldr q7, [x21, #0x0]\n"
       "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "blt 160f\n"
       "159:"  // Height 8: Multiply loop: Main loop head
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "add x27, x27, #0x10\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
-      "add x26, x26, #0x10\n"
+      "sub x9, x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x25, x25, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x23, x23, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "fmla v31.4s, v8.4s, v7.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x20, x20, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x28, #0x8\n"
+      "add x21, x21, #0x10\n"
+      "cmp x9, #0x8\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "add x12, x12, #0x40\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "add x12, x12, #0x40\n"
       "ldr q8, [x12, #0x0]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       "fmla v30.4s, v9.4s, v6.s[1]\n"
       "fmla v31.4s, v9.4s, v7.s[1]\n"
+      "ldr q9, [x12, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
       "fmla v29.4s, v10.4s, v5.s[2]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       "fmla v30.4s, v10.4s, v6.s[2]\n"
       "fmla v31.4s, v10.4s, v7.s[2]\n"
+      "ldr q10, [x12, #0x20]\n"
       "fmla v24.4s, v11.4s, v0.s[3]\n"
-      "ldr q0, [x27, #0x0]\n"
+      "ldr q0, [x28, #0x0]\n"
       "fmla v25.4s, v11.4s, v1.s[3]\n"
-      "ldr q1, [x26, #0x0]\n"
+      "ldr q1, [x27, #0x0]\n"
       "fmla v26.4s, v11.4s, v2.s[3]\n"
-      "ldr q2, [x25, #0x0]\n"
+      "ldr q2, [x26, #0x0]\n"
       "fmla v27.4s, v11.4s, v3.s[3]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "fmla v28.4s, v11.4s, v4.s[3]\n"
-      "ldr q4, [x23, #0x0]\n"
+      "ldr q4, [x24, #0x0]\n"
       "fmla v29.4s, v11.4s, v5.s[3]\n"
-      "ldr q5, [x22, #0x0]\n"
+      "ldr q5, [x23, #0x0]\n"
       "fmla v30.4s, v11.4s, v6.s[3]\n"
-      "ldr q6, [x21, #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       "fmla v31.4s, v11.4s, v7.s[3]\n"
-      "ldr q7, [x20, #0x0]\n"
+      "ldr q7, [x21, #0x0]\n"
+      "ldr q11, [x12, #0x30]\n"
       "bge 159b\n"
       "160:"  // Height 8: Multiply loop: Single iteration only
       "fmla v24.4s, v8.4s, v0.s[0]\n"
-      "ldr q9, [x12, #0x10]\n"
-      "sub x28, x28, #0x4\n"
       "fmla v25.4s, v8.4s, v1.s[0]\n"
-      "ldr q10, [x12, #0x20]\n"
+      "add x28, x28, #0x10\n"
       "add x27, x27, #0x10\n"
       "fmla v26.4s, v8.4s, v2.s[0]\n"
-      "ldr q11, [x12, #0x30]\n"
-      "add x26, x26, #0x10\n"
       "fmla v27.4s, v8.4s, v3.s[0]\n"
-      "prfm pldl1keep, [x27, #0x80]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
       "fmla v28.4s, v8.4s, v4.s[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      "add x24, x24, #0x10\n"
       "fmla v29.4s, v8.4s, v5.s[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       "fmla v30.4s, v8.4s, v6.s[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x22, x22, #0x10\n"
       "fmla v31.4s, v8.4s, v7.s[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x22, x22, #0x10\n"
       "add x21, x21, #0x10\n"
       "fmla v24.4s, v9.4s, v0.s[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x20, x20, #0x10\n"
       "fmla v25.4s, v9.4s, v1.s[1]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "add x12, x12, #0x40\n"
+      "sub x9, x9, #0x4\n"
+      "prfm pldl1keep, [x28, #0x80]\n"
       "fmla v26.4s, v9.4s, v2.s[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "fmla v27.4s, v9.4s, v3.s[1]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "fmla v28.4s, v9.4s, v4.s[1]\n"
       "fmla v29.4s, v9.4s, v5.s[1]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "fmla v30.4s, v9.4s, v6.s[1]\n"
       "fmla v31.4s, v9.4s, v7.s[1]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "fmla v24.4s, v10.4s, v0.s[2]\n"
       "fmla v25.4s, v10.4s, v1.s[2]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x12, x12, #0x40\n"
       "fmla v26.4s, v10.4s, v2.s[2]\n"
       "fmla v27.4s, v10.4s, v3.s[2]\n"
       "fmla v28.4s, v10.4s, v4.s[2]\n"
@@ -1946,132 +1946,132 @@ void a64_hybrid_fp32_mla_8x4 (
       "fmla v30.4s, v11.4s, v6.s[3]\n"
       "fmla v31.4s, v11.4s, v7.s[3]\n"
       "161:"  // Height 8: Multiply loop: Main loop skip
-      "cbz x28, 163f\n"
+      "cbz x9, 163f\n"
       "162:"  // Height 8: Multiply loop: Odd block loop
-      "ldr s0, [x27], #0x4\n"
-      "sub x28, x28, #0x1\n"
-      "ldr s1, [x26], #0x4\n"
-      "ldr s2, [x25], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x23], #0x4\n"
-      "ldr s5, [x22], #0x4\n"
-      "ldr s6, [x21], #0x4\n"
-      "ldr s7, [x20], #0x4\n"
+      "ldr s0, [x28], #0x4\n"
+      "ldr s1, [x27], #0x4\n"
+      "sub x9, x9, #0x1\n"
+      "ldr s2, [x26], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x24], #0x4\n"
+      "ldr s5, [x23], #0x4\n"
+      "ldr s6, [x22], #0x4\n"
+      "ldr s7, [x21], #0x4\n"
       "ldr q12, [x12, #0x0]\n"
       "fmla v24.4s, v12.4s, v0.s[0]\n"
-      "add x12, x12, #0x10\n"
       "fmla v25.4s, v12.4s, v1.s[0]\n"
+      "add x12, x12, #0x10\n"
       "fmla v26.4s, v12.4s, v2.s[0]\n"
       "fmla v27.4s, v12.4s, v3.s[0]\n"
       "fmla v28.4s, v12.4s, v4.s[0]\n"
       "fmla v29.4s, v12.4s, v5.s[0]\n"
       "fmla v30.4s, v12.4s, v6.s[0]\n"
       "fmla v31.4s, v12.4s, v7.s[0]\n"
-      "cbnz x28, 162b\n"
+      "cbnz x9, 162b\n"
       "163:"  // Height 8: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 156b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x10, #0x0]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "prfm pstl1keep, [x26, #0x0]\n"
-      "add x25, x26, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "prfm pstl1keep, [x25, #0x0]\n"
-      "add x24, x25, x19, LSL #2\n"
       "prfm pstl1keep, [x24, #0x0]\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbz %x[flags], #1, 164f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v17.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v16.4s }, [x19]\n"
-      "fmin v24.4s, v24.4s, v16.4s\n"
-      "fmin v25.4s, v25.4s, v16.4s\n"
-      "fmin v26.4s, v26.4s, v16.4s\n"
-      "fmin v27.4s, v27.4s, v16.4s\n"
-      "fmax v24.4s, v24.4s, v17.4s\n"
-      "fmax v25.4s, v25.4s, v17.4s\n"
-      "fmax v26.4s, v26.4s, v17.4s\n"
-      "fmax v27.4s, v27.4s, v17.4s\n"
-      "fmin v28.4s, v28.4s, v16.4s\n"
-      "fmin v29.4s, v29.4s, v16.4s\n"
-      "fmin v30.4s, v30.4s, v16.4s\n"
-      "fmax v28.4s, v28.4s, v17.4s\n"
-      "fmax v29.4s, v29.4s, v17.4s\n"
-      "fmax v30.4s, v30.4s, v17.4s\n"
-      "fmin v31.4s, v31.4s, v16.4s\n"
-      "fmax v31.4s, v31.4s, v17.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v17.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v16.4s }, [x20]\n"
+      "fmin v24.4s, v24.4s, v17.4s\n"
+      "fmin v25.4s, v25.4s, v17.4s\n"
+      "fmin v26.4s, v26.4s, v17.4s\n"
+      "fmin v27.4s, v27.4s, v17.4s\n"
+      "fmin v28.4s, v28.4s, v17.4s\n"
+      "fmin v29.4s, v29.4s, v17.4s\n"
+      "fmin v30.4s, v30.4s, v17.4s\n"
+      "fmin v31.4s, v31.4s, v17.4s\n"
+      "fmax v24.4s, v24.4s, v16.4s\n"
+      "fmax v25.4s, v25.4s, v16.4s\n"
+      "fmax v26.4s, v26.4s, v16.4s\n"
+      "fmax v27.4s, v27.4s, v16.4s\n"
+      "fmax v28.4s, v28.4s, v16.4s\n"
+      "fmax v29.4s, v29.4s, v16.4s\n"
+      "fmax v30.4s, v30.4s, v16.4s\n"
+      "fmax v31.4s, v31.4s, v16.4s\n"
       "164:"  // Height 8: No activation
       "cmp x13, #0x4\n"
       "bge 167f\n"
       "tbz x13, #1, 165f\n"
-      "str d24, [x10], #0x8\n"
-      "str d25, [x26], #0x8\n"
-      "str d26, [x25], #0x8\n"
-      "str d27, [x24], #0x8\n"
-      "str d28, [x23], #0x8\n"
-      "str d29, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d31, [x20], #0x8\n"
+      "str d24, [x11], #0x8\n"
+      "str d25, [x27], #0x8\n"
+      "str d26, [x26], #0x8\n"
+      "str d27, [x25], #0x8\n"
+      "str d28, [x24], #0x8\n"
+      "str d29, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d31, [x21], #0x8\n"
       "tbz x13, #0, 166f\n"
-      "st1 { v24.s }[2], [x10]\n"
-      "st1 { v25.s }[2], [x26]\n"
-      "st1 { v26.s }[2], [x25]\n"
-      "st1 { v27.s }[2], [x24]\n"
-      "st1 { v28.s }[2], [x23]\n"
-      "st1 { v29.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v31.s }[2], [x20]\n"
+      "st1 { v24.s }[2], [x11]\n"
+      "st1 { v25.s }[2], [x27]\n"
+      "st1 { v26.s }[2], [x26]\n"
+      "st1 { v27.s }[2], [x25]\n"
+      "st1 { v28.s }[2], [x24]\n"
+      "st1 { v29.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v31.s }[2], [x21]\n"
       "b 166f\n"
       "165:"  // Height 8: Partial direct writeback: partial_1_0
-      "str s24, [x10, #0x0]\n"
-      "str s25, [x26, #0x0]\n"
-      "str s26, [x25, #0x0]\n"
-      "str s27, [x24, #0x0]\n"
-      "str s28, [x23, #0x0]\n"
-      "str s29, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s31, [x20, #0x0]\n"
+      "str s24, [x11, #0x0]\n"
+      "str s25, [x27, #0x0]\n"
+      "str s26, [x26, #0x0]\n"
+      "str s27, [x25, #0x0]\n"
+      "str s28, [x24, #0x0]\n"
+      "str s29, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s31, [x21, #0x0]\n"
       "166:"  // Height 8: Partial direct writeback: Done
       "b 168f\n"
       "167:"  // Height 8: Full writeback
-      "str q24, [x10, #0x0]\n"
-      "add x10, x10, #0x10\n"
-      "str q25, [x26, #0x0]\n"
-      "str q26, [x25, #0x0]\n"
-      "str q27, [x24, #0x0]\n"
-      "str q28, [x23, #0x0]\n"
-      "str q29, [x22, #0x0]\n"
-      "str q30, [x21, #0x0]\n"
-      "str q31, [x20, #0x0]\n"
+      "str q24, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q25, [x27, #0x0]\n"
+      "str q26, [x26, #0x0]\n"
+      "str q27, [x25, #0x0]\n"
+      "str q28, [x24, #0x0]\n"
+      "str q29, [x23, #0x0]\n"
+      "str q30, [x22, #0x0]\n"
+      "str q31, [x21, #0x0]\n"
       "168:"  // Height 8: Writeback done
       "subs x13, x13, #0x4\n"
       "bgt 149b\n"
       "subs %x[M], %x[M], #0x8\n"
       "beq 170f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 169f\n"
-      "add x20, x20, #0x8\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x8\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "169:"  // Update direct input
-      "mov x19, #0x20\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x20\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "170:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp
index 76c2688291..a0ea96822a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -100,27 +100,27 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "cmp %x[M], #0x2\n"
       "bgt 87f\n"
       "beq 44f\n"
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[bias]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x27, 3f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 3f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x27, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x27, #0x20]\n"
-      "ldr q11, [x27, #0x30]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x27, #0x40]\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q13, [x27, #0x50]\n"
-      "add x27, x27, #0x60\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "add x10, x10, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -131,98 +131,98 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "cmp x9, #0x18\n"
       "bge 16f\n"
       "tbz x9, #4, 7f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
       "tbz x9, #2, 5f\n"
-      "ld1 { v13.4s }, [x26], #0x10\n"
+      "ld1 { v13.4s }, [x27], #0x10\n"
       "tbz x9, #1, 4f\n"
-      "mov x19, #0x58\n"
-      "ldr d20, [x26], #0x8\n"
+      "ldr d20, [x27], #0x8\n"
+      "mov x20, #0x58\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v20.s }[2], [x26]\n"
+      "ld1 { v20.s }[2], [x27]\n"
       "b 15f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 15f\n"
-      "ldr s20, [x26, #0x0]\n"
+      "ldr s20, [x27, #0x0]\n"
       "b 15f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_16
       "tbz x9, #1, 6f\n"
-      "ldr d13, [x26], #0x8\n"
-      "mov x19, #0x48\n"
+      "ldr d13, [x27], #0x8\n"
+      "mov x20, #0x48\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v13.s }[2], [x26]\n"
+      "ld1 { v13.s }[2], [x27]\n"
       "b 15f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 15f\n"
-      "ldr s13, [x26, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
       "b 15f\n"
       "7:"  // Height 1: Partial accumulate: partial_8_0
       "tbz x9, #3, 11f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
       "tbz x9, #2, 9f\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
       "tbz x9, #1, 8f\n"
-      "mov x19, #0x38\n"
-      "ldr d12, [x26], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v12.s }[2], [x26]\n"
+      "ld1 { v12.s }[2], [x27]\n"
       "b 15f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 15f\n"
-      "ldr s12, [x26, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
       "b 15f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x9, #1, 10f\n"
-      "ldr d11, [x26], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x27], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v11.s }[2], [x26]\n"
+      "ld1 { v11.s }[2], [x27]\n"
       "b 15f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 15f\n"
-      "ldr s11, [x26, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
       "b 15f\n"
       "11:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x9, #2, 13f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
       "tbz x9, #1, 12f\n"
-      "ldr d10, [x26], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d10, [x27], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v10.s }[2], [x26]\n"
+      "ld1 { v10.s }[2], [x27]\n"
       "b 15f\n"
       "12:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 15f\n"
-      "ldr s10, [x26, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
       "b 15f\n"
       "13:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x9, #1, 14f\n"
-      "ldr d9, [x26], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x9, #0, 15f\n"
-      "ld1 { v9.s }[2], [x26]\n"
+      "ld1 { v9.s }[2], [x27]\n"
       "b 15f\n"
       "14:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x27, #0x0]\n"
+      "mov x20, #0x0\n"
       "15:"  // Height 1: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 17f\n"
       "16:"  // Height 1: full accumulate
-      "ldr q9, [x26, #0x0]\n"
-      "ldr q10, [x26, #0x10]\n"
-      "ldr q11, [x26, #0x20]\n"
-      "ldr q12, [x26, #0x30]\n"
-      "ldr q13, [x26, #0x40]\n"
-      "ldr q20, [x26, #0x50]\n"
+      "ldr q9, [x27, #0x0]\n"
+      "ldr q10, [x27, #0x10]\n"
+      "ldr q11, [x27, #0x20]\n"
+      "ldr q12, [x27, #0x30]\n"
+      "ldr q13, [x27, #0x40]\n"
+      "ldr q20, [x27, #0x50]\n"
       "17:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -251,264 +251,264 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "19:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "20:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 21f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 22f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 22f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 22f\n"
       "21:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "22:"  // Height 1: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 25f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "cmp x24, #0x8\n"
-      "blt 24f\n"
-      "23:"  // Height 1: Multiply loop: Main loop head
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       "ldr q4, [x28, #0x0]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x10]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "blt 24f\n"
+      "23:"  // Height 1: Multiply loop: Main loop head
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       "ldr q4, [x28, #0x40]\n"
-      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
+      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
-      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
+      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       "ldr q5, [x28, #0x90]\n"
+      ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0xa0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x28, x28, #0xc0\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "ldr q6, [x28, #0x20]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ldr q7, [x28, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       "bge 23b\n"
       "24:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ldr q4, [x28, #0x0]\n"
-      "sub x24, x24, #0x4\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q5, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q4, [x28, #0x40]\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
-      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
+      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       "ldr q4, [x28, #0x80]\n"
+      ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0xa0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x28, x28, #0xc0\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       "25:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 28f\n"
-      "cbz x24, 28f\n"
-      "tbz x24, #1, 26f\n"
-      "ldr d0, [x23], #0x8\n"
-      "tbz x24, #0, 27f\n"
-      "ld1 { v0.s }[2], [x23]\n"
+      "cbz x25, 28f\n"
+      "cbz x25, 28f\n"
+      "tbz x25, #1, 26f\n"
+      "ldr d0, [x24], #0x8\n"
+      "tbz x25, #0, 27f\n"
+      "ld1 { v0.s }[2], [x24]\n"
       "b 27f\n"
       "26:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
       "27:"  // Height 1: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q4, [x28, #0x0]\n"
       "ldr q5, [x28, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       "ldr q6, [x28, #0x20]\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q7, [x28, #0x30]\n"
-      "ldr q4, [x28, #0x40]\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "ldr q4, [x28, #0x40]\n"
       "ldr q5, [x28, #0x50]\n"
-      "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
+      "ldr q6, [x28, #0x60]\n"
       "ldr q7, [x28, #0x70]\n"
-      "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
-      "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q4, [x28, #0x80]\n"
+      "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
+      "ldr q6, [x28, #0xa0]\n"
+      "ldr q7, [x28, #0xb0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x28, x28, #0xc0\n"
       "28:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 20b\n"
       "uzp1 v8.2d, v8.2d, v14.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
       "uzp1 v9.2d, v9.2d, v15.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp1 v10.2d, v10.2d, v16.2d\n"
       "uzp1 v11.2d, v11.2d, v17.2d\n"
       "uzp1 v12.2d, v12.2d, v18.2d\n"
       "uzp1 v13.2d, v13.2d, v19.2d\n"
       "tbz %x[flags], #1, 29f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
       "29:"  // Height 1: No activation
       "cmp x9, #0x18\n"
       "bge 42f\n"
       "tbz x9, #4, 33f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
-      "st1 { v11.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
+      "st1 { v11.4s }, [x27], #0x10\n"
       "tbz x9, #2, 31f\n"
-      "st1 { v12.4s }, [x26], #0x10\n"
+      "st1 { v12.4s }, [x27], #0x10\n"
       "tbz x9, #1, 30f\n"
-      "str d13, [x26], #0x8\n"
+      "str d13, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v13.s }[2], [x26]\n"
+      "st1 { v13.s }[2], [x27]\n"
       "b 41f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_20
       "tbz x9, #0, 41f\n"
-      "str s13, [x26, #0x0]\n"
+      "str s13, [x27, #0x0]\n"
       "b 41f\n"
       "31:"  // Height 1: Partial direct writeback: partial_2_16
       "tbz x9, #1, 32f\n"
-      "str d12, [x26], #0x8\n"
+      "str d12, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v12.s }[2], [x26]\n"
+      "st1 { v12.s }[2], [x27]\n"
       "b 41f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_16
       "tbz x9, #0, 41f\n"
-      "str s12, [x26, #0x0]\n"
+      "str s12, [x27, #0x0]\n"
       "b 41f\n"
       "33:"  // Height 1: Partial direct writeback: partial_8_0
       "tbz x9, #3, 37f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
-      "st1 { v9.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
+      "st1 { v9.4s }, [x27], #0x10\n"
       "tbz x9, #2, 35f\n"
-      "st1 { v10.4s }, [x26], #0x10\n"
+      "st1 { v10.4s }, [x27], #0x10\n"
       "tbz x9, #1, 34f\n"
-      "str d11, [x26], #0x8\n"
+      "str d11, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v11.s }[2], [x26]\n"
+      "st1 { v11.s }[2], [x27]\n"
       "b 41f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 41f\n"
-      "str s11, [x26, #0x0]\n"
+      "str s11, [x27, #0x0]\n"
       "b 41f\n"
       "35:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 36f\n"
-      "str d10, [x26], #0x8\n"
+      "str d10, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v10.s }[2], [x26]\n"
+      "st1 { v10.s }[2], [x27]\n"
       "b 41f\n"
       "36:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 41f\n"
-      "str s10, [x26, #0x0]\n"
+      "str s10, [x27, #0x0]\n"
       "b 41f\n"
       "37:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 39f\n"
-      "st1 { v8.4s }, [x26], #0x10\n"
+      "st1 { v8.4s }, [x27], #0x10\n"
       "tbz x9, #1, 38f\n"
-      "str d9, [x26], #0x8\n"
+      "str d9, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v9.s }[2], [x26]\n"
+      "st1 { v9.s }[2], [x27]\n"
       "b 41f\n"
       "38:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 41f\n"
-      "str s9, [x26, #0x0]\n"
+      "str s9, [x27, #0x0]\n"
       "b 41f\n"
       "39:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 40f\n"
-      "str d8, [x26], #0x8\n"
+      "str d8, [x27], #0x8\n"
       "tbz x9, #0, 41f\n"
-      "st1 { v8.s }[2], [x26]\n"
+      "st1 { v8.s }[2], [x27]\n"
       "b 41f\n"
       "40:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x26, #0x0]\n"
+      "str s8, [x27, #0x0]\n"
       "41:"  // Height 1: Partial direct writeback: Done
       "b 43f\n"
       "42:"  // Height 1: Full writeback
-      "str q8, [x26, #0x0]\n"
-      "str q9, [x26, #0x10]\n"
-      "str q10, [x26, #0x20]\n"
-      "str q11, [x26, #0x30]\n"
-      "str q12, [x26, #0x40]\n"
-      "str q13, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
+      "str q8, [x27, #0x0]\n"
+      "str q9, [x27, #0x10]\n"
+      "str q10, [x27, #0x20]\n"
+      "str q11, [x27, #0x30]\n"
+      "str q12, [x27, #0x40]\n"
+      "str q13, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
       "43:"  // Height 1: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 2b\n"
       "b 174f\n"
       "44:"  // Height 2
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "45:"  // Height 2: Column loop
-      "cbz x27, 46f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 46f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x27, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x27, #0x20]\n"
-      "ldr q11, [x27, #0x30]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
-      "ldr q12, [x27, #0x40]\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
-      "ldr q13, [x27, #0x50]\n"
-      "add x27, x27, #0x60\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "add x10, x10, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
@@ -516,136 +516,136 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "b 62f\n"
       "46:"  // Height 2: no bias
       "tbz %x[flags], #0, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
+      "add x23, x27, x20, LSL #2\n"
       "bge 59f\n"
       "tbz x9, #4, 50f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
       "tbz x9, #2, 48f\n"
-      "ld1 { v13.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
+      "ld1 { v13.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
       "tbz x9, #1, 47f\n"
-      "mov x19, #0x58\n"
-      "ldr d20, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
+      "ldr d20, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v20.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "ld1 { v20.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 58f\n"
       "47:"  // Height 2: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 58f\n"
-      "ldr s20, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "ldr s20, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 58f\n"
       "48:"  // Height 2: Partial accumulate: partial_2_16
       "tbz x9, #1, 49f\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 58f\n"
       "49:"  // Height 2: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 58f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 58f\n"
       "50:"  // Height 2: Partial accumulate: partial_8_0
       "tbz x9, #3, 54f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
       "tbz x9, #2, 52f\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
       "tbz x9, #1, 51f\n"
-      "mov x19, #0x38\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 58f\n"
       "51:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 58f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 58f\n"
       "52:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x9, #1, 53f\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 58f\n"
       "53:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 58f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
       "b 58f\n"
       "54:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x9, #2, 56f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
       "tbz x9, #1, 55f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
       "b 58f\n"
       "55:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 58f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
       "b 58f\n"
       "56:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x9, #1, 57f\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x9, #0, 58f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
       "b 58f\n"
       "57:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
       "58:"  // Height 2: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 60f\n"
       "59:"  // Height 2: full accumulate
-      "ldr q9, [x26, #0x0]\n"
-      "ldr q10, [x26, #0x10]\n"
-      "ldr q11, [x26, #0x20]\n"
-      "ldr q12, [x26, #0x30]\n"
-      "ldr q13, [x26, #0x40]\n"
-      "ldr q20, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
+      "ldr q9, [x27, #0x0]\n"
+      "ldr q10, [x27, #0x10]\n"
+      "ldr q11, [x27, #0x20]\n"
+      "ldr q12, [x27, #0x30]\n"
+      "ldr q13, [x27, #0x40]\n"
+      "ldr q20, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
       "60:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -674,151 +674,151 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
       "62:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "63:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 64f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 65f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 65f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 65f\n"
       "64:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
       "65:"  // Height 2: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 68f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "cmp x24, #0x8\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 67f\n"
       "66:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q4, [x28, #0x0]\n"
-      "cmp x24, #0x8\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q5, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q4, [x28, #0x40]\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
-      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
+      ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       "ldr q4, [x28, #0x80]\n"
+      ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0xa0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       "ldr q7, [x28, #0xb0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
       "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
+      "ldr q6, [x28, #0x20]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ldr q7, [x28, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       "bge 66b\n"
       "67:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q4, [x28, #0x0]\n"
-      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x28, #0x20]\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
-      "ldr q7, [x28, #0x30]\n"
       "ldr q4, [x28, #0x40]\n"
-      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
+      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       "ldr q7, [x28, #0x70]\n"
+      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       "ldr q6, [x28, #0xa0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
+      "sub x25, x25, #0x4\n"
+      ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x28, x28, #0xc0\n"
       "68:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 71f\n"
-      "cbz x24, 71f\n"
-      "tbz x24, #1, 69f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "tbz x24, #0, 70f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
+      "cbz x25, 71f\n"
+      "cbz x25, 71f\n"
+      "tbz x25, #1, 69f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "tbz x25, #0, 70f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
       "b 70f\n"
       "69:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
       "70:"  // Height 2: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q4, [x28, #0x0]\n"
       "ldr q5, [x28, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x28, #0x40]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
+      "ldr q4, [x28, #0x40]\n"
       "ldr q5, [x28, #0x50]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x28, #0x60]\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
+      "ldr q4, [x28, #0x80]\n"
       "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
-      "ldr q6, [x28, #0xa0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x28, #0xa0]\n"
       "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
+      "add x28, x28, #0xc0\n"
       "71:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 63b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
       "uzp2 v10.2d, v10.2d, v16.2d\n"
       "uzp1 v16.2d, v11.2d, v17.2d\n"
@@ -828,183 +828,183 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v18.2d, v13.2d, v19.2d\n"
       "uzp2 v13.2d, v13.2d, v19.2d\n"
       "tbz %x[flags], #1, 72f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v4.4s, v4.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v4.4s, v4.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v4.4s, v4.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmax v4.4s, v4.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
       "72:"  // Height 2: No activation
       "cmp x9, #0x18\n"
       "bge 85f\n"
       "tbz x9, #4, 76f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
       "tbz x9, #2, 74f\n"
-      "st1 { v17.4s }, [x26], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x27], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
       "tbz x9, #1, 73f\n"
-      "str d18, [x26], #0x8\n"
-      "str d13, [x22], #0x8\n"
+      "str d18, [x27], #0x8\n"
+      "str d13, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v18.s }[2], [x26]\n"
-      "st1 { v13.s }[2], [x22]\n"
+      "st1 { v18.s }[2], [x27]\n"
+      "st1 { v13.s }[2], [x23]\n"
       "b 84f\n"
       "73:"  // Height 2: Partial direct writeback: partial_1_20
       "tbz x9, #0, 84f\n"
-      "str s18, [x26, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
+      "str s18, [x27, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
       "b 84f\n"
       "74:"  // Height 2: Partial direct writeback: partial_2_16
       "tbz x9, #1, 75f\n"
-      "str d17, [x26], #0x8\n"
-      "str d12, [x22], #0x8\n"
+      "str d17, [x27], #0x8\n"
+      "str d12, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v17.s }[2], [x26]\n"
-      "st1 { v12.s }[2], [x22]\n"
+      "st1 { v17.s }[2], [x27]\n"
+      "st1 { v12.s }[2], [x23]\n"
       "b 84f\n"
       "75:"  // Height 2: Partial direct writeback: partial_1_16
       "tbz x9, #0, 84f\n"
-      "str s17, [x26, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
+      "str s17, [x27, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
       "b 84f\n"
       "76:"  // Height 2: Partial direct writeback: partial_8_0
       "tbz x9, #3, 80f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
       "tbz x9, #2, 78f\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
       "tbz x9, #1, 77f\n"
-      "str d16, [x26], #0x8\n"
-      "str d11, [x22], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d11, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v16.s }[2], [x26]\n"
-      "st1 { v11.s }[2], [x22]\n"
+      "st1 { v16.s }[2], [x27]\n"
+      "st1 { v11.s }[2], [x23]\n"
       "b 84f\n"
       "77:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 84f\n"
-      "str s16, [x26, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
+      "str s16, [x27, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
       "b 84f\n"
       "78:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 79f\n"
-      "str d15, [x26], #0x8\n"
-      "str d10, [x22], #0x8\n"
+      "str d15, [x27], #0x8\n"
+      "str d10, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v15.s }[2], [x26]\n"
-      "st1 { v10.s }[2], [x22]\n"
+      "st1 { v15.s }[2], [x27]\n"
+      "st1 { v10.s }[2], [x23]\n"
       "b 84f\n"
       "79:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 84f\n"
-      "str s15, [x26, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
+      "str s15, [x27, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
       "b 84f\n"
       "80:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 82f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
       "tbz x9, #1, 81f\n"
-      "str d14, [x26], #0x8\n"
-      "str d9, [x22], #0x8\n"
+      "str d14, [x27], #0x8\n"
+      "str d9, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v14.s }[2], [x26]\n"
-      "st1 { v9.s }[2], [x22]\n"
+      "st1 { v14.s }[2], [x27]\n"
+      "st1 { v9.s }[2], [x23]\n"
       "b 84f\n"
       "81:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 84f\n"
-      "str s14, [x26, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
+      "str s14, [x27, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
       "b 84f\n"
       "82:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 83f\n"
-      "str d4, [x26], #0x8\n"
-      "str d8, [x22], #0x8\n"
+      "str d4, [x27], #0x8\n"
+      "str d8, [x23], #0x8\n"
       "tbz x9, #0, 84f\n"
-      "st1 { v4.s }[2], [x26]\n"
-      "st1 { v8.s }[2], [x22]\n"
+      "st1 { v4.s }[2], [x27]\n"
+      "st1 { v8.s }[2], [x23]\n"
       "b 84f\n"
       "83:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s4, [x26, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
+      "str s4, [x27, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
       "84:"  // Height 2: Partial direct writeback: Done
       "b 86f\n"
       "85:"  // Height 2: Full writeback
-      "str q4, [x26, #0x0]\n"
-      "str q14, [x26, #0x10]\n"
-      "str q15, [x26, #0x20]\n"
-      "str q16, [x26, #0x30]\n"
-      "str q17, [x26, #0x40]\n"
-      "str q18, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
+      "str q4, [x27, #0x0]\n"
+      "str q14, [x27, #0x10]\n"
+      "str q15, [x27, #0x20]\n"
+      "str q16, [x27, #0x30]\n"
+      "str q17, [x27, #0x40]\n"
+      "str q18, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
       "86:"  // Height 2: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 45b\n"
       "b 174f\n"
       "87:"  // Height 3
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
       "88:"  // Height 3: Column loop
-      "cbz x27, 89f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 89f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x27, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x27, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x27, #0x30]\n"
-      "mov v26.16b, v14.16b\n"
-      "ldr q12, [x27, #0x40]\n"
-      "ldr q13, [x27, #0x50]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
-      "add x27, x27, #0x60\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "add x10, x10, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
       "zip1 v13.2d, v13.2d, v13.2d\n"
+      "mov v20.16b, v8.16b\n"
+      "mov v26.16b, v14.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v27.16b, v15.16b\n"
       "mov v22.16b, v10.16b\n"
@@ -1018,170 +1018,170 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "b 105f\n"
       "89:"  // Height 3: no bias
       "tbz %x[flags], #0, 104f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 102f\n"
       "tbz x9, #4, 93f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
       "tbz x9, #2, 91f\n"
-      "ld1 { v13.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v13.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x9, #1, 90f\n"
-      "mov x19, #0x58\n"
-      "ldr d20, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d4, [x21], #0x8\n"
+      "ldr d20, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d4, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v20.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
+      "ld1 { v20.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
       "b 101f\n"
       "90:"  // Height 3: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 101f\n"
-      "ldr s20, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
+      "ldr s20, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
       "b 101f\n"
       "91:"  // Height 3: Partial accumulate: partial_2_16
       "tbz x9, #1, 92f\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 101f\n"
       "92:"  // Height 3: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 101f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "b 101f\n"
       "93:"  // Height 3: Partial accumulate: partial_8_0
       "tbz x9, #3, 97f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
       "tbz x9, #2, 95f\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
       "tbz x9, #1, 94f\n"
-      "mov x19, #0x38\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
       "b 101f\n"
       "94:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 101f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
       "b 101f\n"
       "95:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x9, #1, 96f\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d23, [x21], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d23, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 101f\n"
       "96:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 101f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 101f\n"
       "97:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x9, #2, 99f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
       "tbz x9, #1, 98f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d22, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 101f\n"
       "98:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 101f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 101f\n"
       "99:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x9, #1, 100f\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d21, [x21], #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d21, [x22], #0x8\n"
       "tbz x9, #0, 101f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 101f\n"
       "100:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s21, [x22, #0x0]\n"
       "101:"  // Height 3: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 103f\n"
       "102:"  // Height 3: full accumulate
-      "ldr q9, [x26, #0x0]\n"
-      "ldr q10, [x26, #0x10]\n"
-      "ldr q11, [x26, #0x20]\n"
-      "ldr q12, [x26, #0x30]\n"
-      "ldr q13, [x26, #0x40]\n"
-      "ldr q20, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q21, [x21, #0x0]\n"
-      "ldr q22, [x21, #0x10]\n"
-      "ldr q23, [x21, #0x20]\n"
-      "ldr q24, [x21, #0x30]\n"
-      "ldr q25, [x21, #0x40]\n"
-      "ldr q4, [x21, #0x50]\n"
+      "ldr q9, [x27, #0x0]\n"
+      "ldr q10, [x27, #0x10]\n"
+      "ldr q11, [x27, #0x20]\n"
+      "ldr q12, [x27, #0x30]\n"
+      "ldr q13, [x27, #0x40]\n"
+      "ldr q20, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q21, [x22, #0x0]\n"
+      "ldr q22, [x22, #0x10]\n"
+      "ldr q23, [x22, #0x20]\n"
+      "ldr q24, [x22, #0x30]\n"
+      "ldr q25, [x22, #0x40]\n"
+      "ldr q4, [x22, #0x50]\n"
       "103:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -1234,67 +1234,68 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "105:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "106:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 107f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 108f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 108f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 108f\n"
       "107:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "108:"  // Height 3: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 111f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "cmp x24, #0x8\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 110f\n"
       "109:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q4, [x28, #0x0]\n"
-      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x28, #0x20]\n"
+      "sub x25, x25, #0x4\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "ldr q4, [x28, #0x40]\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "cmp x25, #0x8\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5c  // bfmmla v28.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x90]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec57  // bfmmla v23.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0xa0]\n"
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
@@ -1303,41 +1304,40 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec59  // bfmmla v25.4s, v2.8h, v6.8h\n"
+      "ldr q6, [x28, #0x20]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q7, [x28, #0x30]\n"
       "bge 109b\n"
       "110:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "ldr q4, [x28, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x28, #0x20]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "sub x25, x25, #0x4\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x40]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
@@ -1359,37 +1359,37 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "111:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 114f\n"
-      "cbz x24, 114f\n"
-      "tbz x24, #1, 112f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "ldr d2, [x21], #0x8\n"
-      "tbz x24, #0, 113f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
-      "ld1 { v2.s }[2], [x21]\n"
+      "cbz x25, 114f\n"
+      "cbz x25, 114f\n"
+      "tbz x25, #1, 112f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "ldr d2, [x22], #0x8\n"
+      "tbz x25, #0, 113f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
+      "ld1 { v2.s }[2], [x22]\n"
       "b 113f\n"
       "112:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
-      "ldr s2, [x21, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
+      "ldr s2, [x22, #0x0]\n"
       "113:"  // Height 3: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q4, [x28, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q5, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
+      "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
-      ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x40]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
+      ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
@@ -1406,8 +1406,8 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec11  // bfmmla v17.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5d  // bfmmla v29.4s, v2.8h, v7.8h\n"
       "ldr q7, [x28, #0xb0]\n"
-      "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
+      "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
@@ -1416,21 +1416,21 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "114:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 106b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v16.2d\n"
       "uzp1 v16.2d, v11.2d, v17.2d\n"
       "uzp2 v11.2d, v11.2d, v17.2d\n"
@@ -1445,231 +1445,231 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v24.2d, v24.2d, v30.2d\n"
       "uzp1 v25.2d, v25.2d, v31.2d\n"
       "tbz %x[flags], #1, 115f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v4.4s, v4.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v4.4s, v4.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v4.4s, v4.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmax v4.4s, v4.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
       "115:"  // Height 3: No activation
       "cmp x9, #0x18\n"
       "bge 128f\n"
       "tbz x9, #4, 119f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
       "tbz x9, #2, 117f\n"
-      "st1 { v17.4s }, [x26], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v17.4s }, [x27], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x9, #1, 116f\n"
-      "str d18, [x26], #0x8\n"
-      "str d13, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d18, [x27], #0x8\n"
+      "str d13, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v18.s }[2], [x26]\n"
-      "st1 { v13.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v18.s }[2], [x27]\n"
+      "st1 { v13.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 127f\n"
       "116:"  // Height 3: Partial direct writeback: partial_1_20
       "tbz x9, #0, 127f\n"
-      "str s18, [x26, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s18, [x27, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 127f\n"
       "117:"  // Height 3: Partial direct writeback: partial_2_16
       "tbz x9, #1, 118f\n"
-      "str d17, [x26], #0x8\n"
-      "str d12, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d17, [x27], #0x8\n"
+      "str d12, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v17.s }[2], [x26]\n"
-      "st1 { v12.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v17.s }[2], [x27]\n"
+      "st1 { v12.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 127f\n"
       "118:"  // Height 3: Partial direct writeback: partial_1_16
       "tbz x9, #0, 127f\n"
-      "str s17, [x26, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s17, [x27, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "b 127f\n"
       "119:"  // Height 3: Partial direct writeback: partial_8_0
       "tbz x9, #3, 123f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
       "tbz x9, #2, 121f\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
       "tbz x9, #1, 120f\n"
-      "str d16, [x26], #0x8\n"
-      "str d11, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d11, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v16.s }[2], [x26]\n"
-      "st1 { v11.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "st1 { v16.s }[2], [x27]\n"
+      "st1 { v11.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 127f\n"
       "120:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 127f\n"
-      "str s16, [x26, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "str s16, [x27, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 127f\n"
       "121:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 122f\n"
-      "str d15, [x26], #0x8\n"
-      "str d10, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
+      "str d15, [x27], #0x8\n"
+      "str d10, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v15.s }[2], [x26]\n"
-      "st1 { v10.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "st1 { v15.s }[2], [x27]\n"
+      "st1 { v10.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 127f\n"
       "122:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 127f\n"
-      "str s15, [x26, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "str s15, [x27, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 127f\n"
       "123:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 125f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
       "tbz x9, #1, 124f\n"
-      "str d14, [x26], #0x8\n"
-      "str d9, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
+      "str d14, [x27], #0x8\n"
+      "str d9, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v14.s }[2], [x26]\n"
-      "st1 { v9.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "st1 { v14.s }[2], [x27]\n"
+      "st1 { v9.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 127f\n"
       "124:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 127f\n"
-      "str s14, [x26, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "str s14, [x27, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 127f\n"
       "125:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 126f\n"
-      "str d4, [x26], #0x8\n"
-      "str d8, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
+      "str d4, [x27], #0x8\n"
+      "str d8, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
       "tbz x9, #0, 127f\n"
-      "st1 { v4.s }[2], [x26]\n"
-      "st1 { v8.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "st1 { v4.s }[2], [x27]\n"
+      "st1 { v8.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 127f\n"
       "126:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s4, [x26, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s4, [x27, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "127:"  // Height 3: Partial direct writeback: Done
       "b 129f\n"
       "128:"  // Height 3: Full writeback
-      "str q4, [x26, #0x0]\n"
-      "str q14, [x26, #0x10]\n"
-      "str q15, [x26, #0x20]\n"
-      "str q16, [x26, #0x30]\n"
-      "str q17, [x26, #0x40]\n"
-      "str q18, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x21, #0x40]\n"
-      "str q25, [x21, #0x50]\n"
+      "str q4, [x27, #0x0]\n"
+      "str q14, [x27, #0x10]\n"
+      "str q15, [x27, #0x20]\n"
+      "str q16, [x27, #0x30]\n"
+      "str q17, [x27, #0x40]\n"
+      "str q18, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x22, #0x40]\n"
+      "str q25, [x22, #0x50]\n"
       "129:"  // Height 3: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 88b\n"
       "b 174f\n"
       "130:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x10\n"
+      "mov x10, %x[bias]\n"
       "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[bias]\n"
       "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x10\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "131:"  // Height 4: Column loop
-      "cbz x27, 132f\n"
-      "ldr q8, [x27, #0x0]\n"
+      "cbz x10, 132f\n"
+      "ldr q8, [x10, #0x0]\n"
+      "ldr q9, [x10, #0x10]\n"
       "zip2 v14.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x27, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x27, #0x20]\n"
-      "mov v20.16b, v8.16b\n"
-      "ldr q11, [x27, #0x30]\n"
-      "mov v26.16b, v14.16b\n"
-      "ldr q12, [x27, #0x40]\n"
-      "ldr q13, [x27, #0x50]\n"
+      "ldr q10, [x10, #0x20]\n"
+      "ldr q11, [x10, #0x30]\n"
       "zip2 v15.2d, v9.2d, v9.2d\n"
-      "add x27, x27, #0x60\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
+      "ldr q12, [x10, #0x40]\n"
+      "ldr q13, [x10, #0x50]\n"
       "zip2 v16.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
       "zip2 v17.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "add x10, x10, #0x60\n"
       "zip2 v18.2d, v12.2d, v12.2d\n"
       "zip1 v12.2d, v12.2d, v12.2d\n"
       "zip2 v19.2d, v13.2d, v13.2d\n"
       "zip1 v13.2d, v13.2d, v13.2d\n"
+      "mov v20.16b, v8.16b\n"
+      "mov v26.16b, v14.16b\n"
       "mov v21.16b, v9.16b\n"
       "mov v27.16b, v15.16b\n"
       "mov v22.16b, v10.16b\n"
@@ -1683,204 +1683,204 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "b 148f\n"
       "132:"  // Height 4: no bias
       "tbz %x[flags], #0, 147f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x9, #0x18\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 145f\n"
       "tbz x9, #4, 136f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v12.4s }, [x26], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v24.4s }, [x21], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v12.4s }, [x27], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v24.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x9, #2, 134f\n"
-      "ld1 { v13.4s }, [x26], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v13.4s }, [x27], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x9, #1, 133f\n"
-      "mov x19, #0x58\n"
-      "ldr d20, [x26], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d4, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d20, [x27], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "mov x20, #0x58\n"
+      "ldr d4, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v20.s }[2], [x26]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v20.s }[2], [x27]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 144f\n"
       "133:"  // Height 4: Partial accumulate: partial_1_20
-      "mov x19, #0x50\n"
+      "mov x20, #0x50\n"
       "tbz x9, #0, 144f\n"
-      "ldr s20, [x26, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s20, [x27, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 144f\n"
       "134:"  // Height 4: Partial accumulate: partial_2_16
       "tbz x9, #1, 135f\n"
-      "ldr d13, [x26], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "mov x19, #0x48\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d13, [x27], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "mov x20, #0x48\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v13.s }[2], [x26]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v13.s }[2], [x27]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 144f\n"
       "135:"  // Height 4: Partial accumulate: partial_1_16
-      "mov x19, #0x40\n"
+      "mov x20, #0x40\n"
       "tbz x9, #0, 144f\n"
-      "ldr s13, [x26, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s13, [x27, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 144f\n"
       "136:"  // Height 4: Partial accumulate: partial_8_0
       "tbz x9, #3, 140f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x26], #0x10\n"
-      "ld1 { v15.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x27], #0x10\n"
+      "ld1 { v15.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
       "tbz x9, #2, 138f\n"
-      "ld1 { v11.4s }, [x26], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v23.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v11.4s }, [x27], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v23.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x9, #1, 137f\n"
-      "mov x19, #0x38\n"
-      "ldr d12, [x26], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d24, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d12, [x27], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v12.s }[2], [x26]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v24.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v12.s }[2], [x27]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v24.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 144f\n"
       "137:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x9, #0, 144f\n"
-      "ldr s12, [x26, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s24, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s12, [x27, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s24, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 144f\n"
       "138:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x9, #1, 139f\n"
-      "ldr d11, [x26], #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d11, [x27], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v11.s }[2], [x26]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x27]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 144f\n"
       "139:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x9, #0, 144f\n"
-      "ldr s11, [x26, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s11, [x27, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "b 144f\n"
       "140:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x9, #2, 142f\n"
-      "ld1 { v9.4s }, [x26], #0x10\n"
-      "ld1 { v14.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x27], #0x10\n"
+      "ld1 { v14.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
       "tbz x9, #1, 141f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x26], #0x8\n"
-      "ldr d15, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
+      "ldr d10, [x27], #0x8\n"
+      "ldr d15, [x23], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v10.s }[2], [x26]\n"
-      "ld1 { v15.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x27]\n"
+      "ld1 { v15.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 144f\n"
       "141:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x9, #0, 144f\n"
-      "ldr s10, [x26, #0x0]\n"
-      "ldr s15, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "ldr s10, [x27, #0x0]\n"
+      "ldr s15, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 144f\n"
       "142:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x9, #1, 143f\n"
-      "ldr d9, [x26], #0x8\n"
-      "ldr d14, [x22], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
+      "ldr d9, [x27], #0x8\n"
+      "ldr d14, [x23], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
       "tbz x9, #0, 144f\n"
-      "ld1 { v9.s }[2], [x26]\n"
-      "ld1 { v14.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x27]\n"
+      "ld1 { v14.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 144f\n"
       "143:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x26, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s14, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "ldr s9, [x27, #0x0]\n"
+      "ldr s14, [x23, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "144:"  // Height 4: Partial accumulate: Done
-      "sub x26, x26, x19\n"
+      "sub x27, x27, x20\n"
       "b 146f\n"
       "145:"  // Height 4: full accumulate
-      "ldr q9, [x26, #0x0]\n"
-      "ldr q10, [x26, #0x10]\n"
-      "ldr q11, [x26, #0x20]\n"
-      "ldr q12, [x26, #0x30]\n"
-      "ldr q13, [x26, #0x40]\n"
-      "ldr q20, [x26, #0x50]\n"
-      "ldr q14, [x22, #0x0]\n"
-      "ldr q15, [x22, #0x10]\n"
-      "ldr q16, [x22, #0x20]\n"
-      "ldr q17, [x22, #0x30]\n"
-      "ldr q18, [x22, #0x40]\n"
-      "ldr q19, [x22, #0x50]\n"
-      "ldr q21, [x21, #0x0]\n"
-      "ldr q22, [x21, #0x10]\n"
-      "ldr q23, [x21, #0x20]\n"
-      "ldr q24, [x21, #0x30]\n"
-      "ldr q25, [x21, #0x40]\n"
-      "ldr q4, [x21, #0x50]\n"
-      "ldr q26, [x20, #0x0]\n"
-      "ldr q27, [x20, #0x10]\n"
-      "ldr q28, [x20, #0x20]\n"
-      "ldr q29, [x20, #0x30]\n"
-      "ldr q30, [x20, #0x40]\n"
-      "ldr q31, [x20, #0x50]\n"
+      "ldr q9, [x27, #0x0]\n"
+      "ldr q10, [x27, #0x10]\n"
+      "ldr q11, [x27, #0x20]\n"
+      "ldr q12, [x27, #0x30]\n"
+      "ldr q13, [x27, #0x40]\n"
+      "ldr q20, [x27, #0x50]\n"
+      "ldr q14, [x23, #0x0]\n"
+      "ldr q15, [x23, #0x10]\n"
+      "ldr q16, [x23, #0x20]\n"
+      "ldr q17, [x23, #0x30]\n"
+      "ldr q18, [x23, #0x40]\n"
+      "ldr q19, [x23, #0x50]\n"
+      "ldr q21, [x22, #0x0]\n"
+      "ldr q22, [x22, #0x10]\n"
+      "ldr q23, [x22, #0x20]\n"
+      "ldr q24, [x22, #0x30]\n"
+      "ldr q25, [x22, #0x40]\n"
+      "ldr q4, [x22, #0x50]\n"
+      "ldr q26, [x21, #0x0]\n"
+      "ldr q27, [x21, #0x10]\n"
+      "ldr q28, [x21, #0x20]\n"
+      "ldr q29, [x21, #0x30]\n"
+      "ldr q30, [x21, #0x40]\n"
+      "ldr q31, [x21, #0x50]\n"
       "146:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v14.2d\n"
       "zip2 v14.2d, v9.2d, v14.2d\n"
@@ -1933,67 +1933,69 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "148:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "149:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 150f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 151f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 151f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 151f\n"
       "150:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "151:"  // Height 4: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "blt 154f\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
-      "cmp x24, #0x8\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "cmp x25, #0x8\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
+      "ld1 { v3.4s }, [x21], #0x10\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
       "blt 153f\n"
       "152:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "cmp x24, #0x8\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v3.4s }, [x20], #0x10\n"
-      "ldr q4, [x28, #0x0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x8\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      "ldr q6, [x28, #0x20]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x40]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v1.4s }, [x23], #0x10\n"
       ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
+      "ld1 { v3.4s }, [x21], #0x10\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
@@ -2008,44 +2010,42 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "add x28, x28, #0xc0\n"
       ".inst 0x6e44ec0c  // bfmmla v12.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec58  // bfmmla v24.4s, v2.8h, v4.8h\n"
+      "ldr q4, [x28, #0x0]\n"
       ".inst 0x6e45ec12  // bfmmla v18.4s, v0.8h, v5.8h\n"
       ".inst 0x6e45ec5e  // bfmmla v30.4s, v2.8h, v5.8h\n"
+      "ldr q5, [x28, #0x10]\n"
       ".inst 0x6e46ec0d  // bfmmla v13.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec59  // bfmmla v25.4s, v2.8h, v6.8h\n"
+      "ldr q6, [x28, #0x20]\n"
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x23], #0x10\n"
+      "ld1 { v0.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x22], #0x10\n"
+      "ldr q7, [x28, #0x30]\n"
       "bge 152b\n"
       "153:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x22], #0x10\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x21], #0x10\n"
-      "ld1 { v3.4s }, [x20], #0x10\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q4, [x28, #0x0]\n"
-      "ldr q5, [x28, #0x10]\n"
+      "sub x25, x25, #0x4\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q6, [x28, #0x20]\n"
-      "ldr q7, [x28, #0x30]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q4, [x28, #0x40]\n"
-      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e45ec0e  // bfmmla v14.4s, v0.8h, v5.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      ".inst 0x6e45ec5a  // bfmmla v26.4s, v2.8h, v5.8h\n"
       "ldr q5, [x28, #0x50]\n"
+      ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
       "ldr q6, [x28, #0x60]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e47ec5b  // bfmmla v27.4s, v2.8h, v7.8h\n"
       "ldr q7, [x28, #0x70]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       "ldr q4, [x28, #0x80]\n"
       ".inst 0x6e45ec10  // bfmmla v16.4s, v0.8h, v5.8h\n"
@@ -2067,32 +2067,32 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "154:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 157f\n"
-      "cbz x24, 157f\n"
-      "tbz x24, #1, 155f\n"
-      "ldr d0, [x23], #0x8\n"
-      "ldr d1, [x22], #0x8\n"
-      "ldr d2, [x21], #0x8\n"
-      "ldr d3, [x20], #0x8\n"
-      "tbz x24, #0, 156f\n"
-      "ld1 { v0.s }[2], [x23]\n"
-      "ld1 { v1.s }[2], [x22]\n"
-      "ld1 { v2.s }[2], [x21]\n"
-      "ld1 { v3.s }[2], [x20]\n"
+      "cbz x25, 157f\n"
+      "cbz x25, 157f\n"
+      "tbz x25, #1, 155f\n"
+      "ldr d0, [x24], #0x8\n"
+      "ldr d1, [x23], #0x8\n"
+      "ldr d2, [x22], #0x8\n"
+      "ldr d3, [x21], #0x8\n"
+      "tbz x25, #0, 156f\n"
+      "ld1 { v0.s }[2], [x24]\n"
+      "ld1 { v1.s }[2], [x23]\n"
+      "ld1 { v2.s }[2], [x22]\n"
+      "ld1 { v3.s }[2], [x21]\n"
       "b 156f\n"
       "155:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x23, #0x0]\n"
-      "ldr s1, [x22, #0x0]\n"
-      "ldr s2, [x21, #0x0]\n"
-      "ldr s3, [x20, #0x0]\n"
+      "ldr s0, [x24, #0x0]\n"
+      "ldr s1, [x23, #0x0]\n"
+      "ldr s2, [x22, #0x0]\n"
+      "ldr s3, [x21, #0x0]\n"
       "156:"  // Height 4: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q4, [x28, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q5, [x28, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q6, [x28, #0x20]\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       "ldr q7, [x28, #0x30]\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
@@ -2128,25 +2128,25 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       ".inst 0x6e47ec13  // bfmmla v19.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec5f  // bfmmla v31.4s, v2.8h, v7.8h\n"
       "157:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 149b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 v4.2d, v8.2d, v14.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v14.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x22, x26, x19, LSL #2\n"
       "uzp1 v14.2d, v9.2d, v15.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v15.2d\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v15.2d, v10.2d, v16.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v16.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp1 v16.2d, v11.2d, v17.2d\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v17.2d\n"
       "uzp1 v17.2d, v12.2d, v18.2d\n"
       "uzp2 v12.2d, v12.2d, v18.2d\n"
@@ -2165,260 +2165,260 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 (
       "uzp1 v30.2d, v25.2d, v31.2d\n"
       "uzp2 v25.2d, v25.2d, v31.2d\n"
       "tbz %x[flags], #1, 158f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v4.4s, v4.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v4.4s, v4.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v4.4s, v4.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmax v4.4s, v4.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
       "158:"  // Height 4: No activation
       "cmp x9, #0x18\n"
       "bge 171f\n"
       "tbz x9, #4, 162f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v16.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v11.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v27.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
-      "st1 { v21.4s }, [x20], #0x10\n"
-      "st1 { v22.4s }, [x20], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v16.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v11.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v27.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v21.4s }, [x21], #0x10\n"
+      "st1 { v22.4s }, [x21], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
       "tbz x9, #2, 160f\n"
-      "st1 { v17.4s }, [x26], #0x10\n"
-      "st1 { v12.4s }, [x22], #0x10\n"
-      "st1 { v29.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
+      "st1 { v17.4s }, [x27], #0x10\n"
+      "st1 { v12.4s }, [x23], #0x10\n"
+      "st1 { v29.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
       "tbz x9, #1, 159f\n"
-      "str d18, [x26], #0x8\n"
-      "str d13, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
+      "str d18, [x27], #0x8\n"
+      "str d13, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v18.s }[2], [x26]\n"
-      "st1 { v13.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "st1 { v18.s }[2], [x27]\n"
+      "st1 { v13.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 170f\n"
       "159:"  // Height 4: Partial direct writeback: partial_1_20
       "tbz x9, #0, 170f\n"
-      "str s18, [x26, #0x0]\n"
-      "str s13, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "str s18, [x27, #0x0]\n"
+      "str s13, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 170f\n"
       "160:"  // Height 4: Partial direct writeback: partial_2_16
       "tbz x9, #1, 161f\n"
-      "str d17, [x26], #0x8\n"
-      "str d12, [x22], #0x8\n"
-      "str d29, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d17, [x27], #0x8\n"
+      "str d12, [x23], #0x8\n"
+      "str d29, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v17.s }[2], [x26]\n"
-      "st1 { v12.s }[2], [x22]\n"
-      "st1 { v29.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "st1 { v17.s }[2], [x27]\n"
+      "st1 { v12.s }[2], [x23]\n"
+      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 170f\n"
       "161:"  // Height 4: Partial direct writeback: partial_1_16
       "tbz x9, #0, 170f\n"
-      "str s17, [x26, #0x0]\n"
-      "str s12, [x22, #0x0]\n"
-      "str s29, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s17, [x27, #0x0]\n"
+      "str s12, [x23, #0x0]\n"
+      "str s29, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "b 170f\n"
       "162:"  // Height 4: Partial direct writeback: partial_8_0
       "tbz x9, #3, 166f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v14.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v9.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
-      "st1 { v21.4s }, [x20], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v14.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v9.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
+      "st1 { v21.4s }, [x21], #0x10\n"
       "tbz x9, #2, 164f\n"
-      "st1 { v15.4s }, [x26], #0x10\n"
-      "st1 { v10.4s }, [x22], #0x10\n"
-      "st1 { v27.4s }, [x21], #0x10\n"
-      "st1 { v22.4s }, [x20], #0x10\n"
+      "st1 { v15.4s }, [x27], #0x10\n"
+      "st1 { v10.4s }, [x23], #0x10\n"
+      "st1 { v27.4s }, [x22], #0x10\n"
+      "st1 { v22.4s }, [x21], #0x10\n"
       "tbz x9, #1, 163f\n"
-      "str d16, [x26], #0x8\n"
-      "str d11, [x22], #0x8\n"
-      "str d28, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d11, [x23], #0x8\n"
+      "str d28, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v16.s }[2], [x26]\n"
-      "st1 { v11.s }[2], [x22]\n"
-      "st1 { v28.s }[2], [x21]\n"
-      "st1 { v23.s }[2], [x20]\n"
+      "st1 { v16.s }[2], [x27]\n"
+      "st1 { v11.s }[2], [x23]\n"
+      "st1 { v28.s }[2], [x22]\n"
+      "st1 { v23.s }[2], [x21]\n"
       "b 170f\n"
       "163:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 170f\n"
-      "str s16, [x26, #0x0]\n"
-      "str s11, [x22, #0x0]\n"
-      "str s28, [x21, #0x0]\n"
-      "str s23, [x20, #0x0]\n"
+      "str s16, [x27, #0x0]\n"
+      "str s11, [x23, #0x0]\n"
+      "str s28, [x22, #0x0]\n"
+      "str s23, [x21, #0x0]\n"
       "b 170f\n"
       "164:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 165f\n"
-      "str d15, [x26], #0x8\n"
-      "str d10, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
-      "str d22, [x20], #0x8\n"
+      "str d15, [x27], #0x8\n"
+      "str d10, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
+      "str d22, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v15.s }[2], [x26]\n"
-      "st1 { v10.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
-      "st1 { v22.s }[2], [x20]\n"
+      "st1 { v15.s }[2], [x27]\n"
+      "st1 { v10.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
+      "st1 { v22.s }[2], [x21]\n"
       "b 170f\n"
       "165:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 170f\n"
-      "str s15, [x26, #0x0]\n"
-      "str s10, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
-      "str s22, [x20, #0x0]\n"
+      "str s15, [x27, #0x0]\n"
+      "str s10, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
+      "str s22, [x21, #0x0]\n"
       "b 170f\n"
       "166:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 168f\n"
-      "st1 { v4.4s }, [x26], #0x10\n"
-      "st1 { v8.4s }, [x22], #0x10\n"
-      "st1 { v19.4s }, [x21], #0x10\n"
-      "st1 { v20.4s }, [x20], #0x10\n"
+      "st1 { v4.4s }, [x27], #0x10\n"
+      "st1 { v8.4s }, [x23], #0x10\n"
+      "st1 { v19.4s }, [x22], #0x10\n"
+      "st1 { v20.4s }, [x21], #0x10\n"
       "tbz x9, #1, 167f\n"
-      "str d14, [x26], #0x8\n"
-      "str d9, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
-      "str d21, [x20], #0x8\n"
+      "str d14, [x27], #0x8\n"
+      "str d9, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
+      "str d21, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v14.s }[2], [x26]\n"
-      "st1 { v9.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
-      "st1 { v21.s }[2], [x20]\n"
+      "st1 { v14.s }[2], [x27]\n"
+      "st1 { v9.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
+      "st1 { v21.s }[2], [x21]\n"
       "b 170f\n"
       "167:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 170f\n"
-      "str s14, [x26, #0x0]\n"
-      "str s9, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
-      "str s21, [x20, #0x0]\n"
+      "str s14, [x27, #0x0]\n"
+      "str s9, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
+      "str s21, [x21, #0x0]\n"
       "b 170f\n"
       "168:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 169f\n"
-      "str d4, [x26], #0x8\n"
-      "str d8, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d20, [x20], #0x8\n"
+      "str d4, [x27], #0x8\n"
+      "str d8, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d20, [x21], #0x8\n"
       "tbz x9, #0, 170f\n"
-      "st1 { v4.s }[2], [x26]\n"
-      "st1 { v8.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v20.s }[2], [x20]\n"
+      "st1 { v4.s }[2], [x27]\n"
+      "st1 { v8.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v20.s }[2], [x21]\n"
       "b 170f\n"
       "169:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s4, [x26, #0x0]\n"
-      "str s8, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s20, [x20, #0x0]\n"
+      "str s4, [x27, #0x0]\n"
+      "str s8, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s20, [x21, #0x0]\n"
       "170:"  // Height 4: Partial direct writeback: Done
       "b 172f\n"
       "171:"  // Height 4: Full writeback
-      "str q4, [x26, #0x0]\n"
-      "str q14, [x26, #0x10]\n"
-      "str q15, [x26, #0x20]\n"
-      "str q16, [x26, #0x30]\n"
-      "str q17, [x26, #0x40]\n"
-      "str q18, [x26, #0x50]\n"
-      "add x26, x26, #0x60\n"
-      "str q8, [x22, #0x0]\n"
-      "str q9, [x22, #0x10]\n"
-      "str q10, [x22, #0x20]\n"
-      "str q11, [x22, #0x30]\n"
-      "str q12, [x22, #0x40]\n"
-      "str q13, [x22, #0x50]\n"
-      "str q19, [x21, #0x0]\n"
-      "str q26, [x21, #0x10]\n"
-      "str q27, [x21, #0x20]\n"
-      "str q28, [x21, #0x30]\n"
-      "str q29, [x21, #0x40]\n"
-      "str q30, [x21, #0x50]\n"
-      "str q20, [x20, #0x0]\n"
-      "str q21, [x20, #0x10]\n"
-      "str q22, [x20, #0x20]\n"
-      "str q23, [x20, #0x30]\n"
-      "str q24, [x20, #0x40]\n"
-      "str q25, [x20, #0x50]\n"
+      "str q4, [x27, #0x0]\n"
+      "str q14, [x27, #0x10]\n"
+      "str q15, [x27, #0x20]\n"
+      "str q16, [x27, #0x30]\n"
+      "str q17, [x27, #0x40]\n"
+      "str q18, [x27, #0x50]\n"
+      "add x27, x27, #0x60\n"
+      "str q8, [x23, #0x0]\n"
+      "str q9, [x23, #0x10]\n"
+      "str q10, [x23, #0x20]\n"
+      "str q11, [x23, #0x30]\n"
+      "str q12, [x23, #0x40]\n"
+      "str q13, [x23, #0x50]\n"
+      "str q19, [x22, #0x0]\n"
+      "str q26, [x22, #0x10]\n"
+      "str q27, [x22, #0x20]\n"
+      "str q28, [x22, #0x30]\n"
+      "str q29, [x22, #0x40]\n"
+      "str q30, [x22, #0x50]\n"
+      "str q20, [x21, #0x0]\n"
+      "str q21, [x21, #0x10]\n"
+      "str q22, [x21, #0x20]\n"
+      "str q23, [x21, #0x30]\n"
+      "str q24, [x21, #0x40]\n"
+      "str q25, [x21, #0x50]\n"
       "172:"  // Height 4: Writeback done
       "subs x9, x9, #0x18\n"
       "bgt 131b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 174f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 173f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "173:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "174:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp
index 19dbf0588e..4993777d62 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -103,23 +103,23 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 71f\n"
       "beq 36f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "cbz x9, 3f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 3f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
-      "add x9, x9, #0x40\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 15f\n"
@@ -128,65 +128,65 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "cmp x11, #0x10\n"
       "bge 12f\n"
       "tbz x11, #3, 7f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #2, 5f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
       "tbz x11, #1, 4f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v16.s }[2], [x28]\n"
+      "ld1 { v16.s }[2], [x9]\n"
       "b 11f\n"
       "4:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 11f\n"
-      "ldr s16, [x28, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
       "b 11f\n"
       "5:"  // Height 1: Partial accumulate: partial_2_8
       "tbz x11, #1, 6f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 11f\n"
       "6:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 11f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 11f\n"
       "7:"  // Height 1: Partial accumulate: partial_4_0
       "tbz x11, #2, 9f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #1, 8f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 11f\n"
       "8:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 11f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 11f\n"
       "9:"  // Height 1: Partial accumulate: partial_2_0
       "tbz x11, #1, 10f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 11f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 11f\n"
       "10:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
+      "ldr s9, [x9, #0x0]\n"
+      "mov x20, #0x0\n"
       "11:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 13f\n"
       "12:"  // Height 1: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
       "13:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -207,86 +207,86 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "15:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 18f\n"
       "17:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "18:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 21f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "cmp x27, #0x8\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 20f\n"
       "19:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
       "bge 19b\n"
       "20:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q7, [x10, #0x10]\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "sub x27, x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x80\n"
       "21:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 24f\n"
-      "cbz x26, 24f\n"
-      "tbz x26, #1, 22f\n"
-      "ldr d0, [x25], #0x8\n"
-      "tbz x26, #0, 23f\n"
-      "ld1 { v0.s }[2], [x25]\n"
+      "cbz x27, 24f\n"
+      "cbz x27, 24f\n"
+      "tbz x27, #1, 22f\n"
+      "ldr d0, [x26], #0x8\n"
+      "tbz x27, #0, 23f\n"
+      "ld1 { v0.s }[2], [x26]\n"
       "b 23f\n"
       "22:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
@@ -299,198 +299,198 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x80\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
       "uzp1 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
       "25:"  // Height 1: No activation
       "cmp x11, #0x10\n"
       "bge 34f\n"
       "tbz x11, #3, 29f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
       "tbz x11, #2, 27f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
       "tbz x11, #1, 26f\n"
-      "str d11, [x28], #0x8\n"
+      "str d11, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 33f\n"
       "26:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x11, #0, 33f\n"
-      "str s11, [x28, #0x0]\n"
+      "str s11, [x9, #0x0]\n"
       "b 33f\n"
       "27:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x11, #1, 28f\n"
-      "str d10, [x28], #0x8\n"
+      "str d10, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 33f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x11, #0, 33f\n"
-      "str s10, [x28, #0x0]\n"
+      "str s10, [x9, #0x0]\n"
       "b 33f\n"
       "29:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x11, #2, 31f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
       "tbz x11, #1, 30f\n"
-      "str d9, [x28], #0x8\n"
+      "str d9, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 33f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x11, #0, 33f\n"
-      "str s9, [x28, #0x0]\n"
+      "str s9, [x9, #0x0]\n"
       "b 33f\n"
       "31:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x11, #1, 32f\n"
-      "str d8, [x28], #0x8\n"
+      "str d8, [x9], #0x8\n"
       "tbz x11, #0, 33f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 33f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "33:"  // Height 1: Partial direct writeback: Done
       "b 35f\n"
       "34:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "35:"  // Height 1: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 212f\n"
       "36:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "37:"  // Height 2: Column loop
-      "cbz x9, 38f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 38f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "ldr q11, [x9, #0x30]\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
-      "add x9, x9, #0x40\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
       "b 50f\n"
       "38:"  // Height 2: no bias
       "tbz %x[flags], #0, 49f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "bge 47f\n"
       "tbz x11, #3, 42f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
       "tbz x11, #2, 40f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
       "tbz x11, #1, 39f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
       "b 46f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 46f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
       "b 46f\n"
       "40:"  // Height 2: Partial accumulate: partial_2_8
       "tbz x11, #1, 41f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
       "b 46f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 46f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
       "b 46f\n"
       "42:"  // Height 2: Partial accumulate: partial_4_0
       "tbz x11, #2, 44f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
       "tbz x11, #1, 43f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
       "b 46f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 46f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
       "b 46f\n"
       "44:"  // Height 2: Partial accumulate: partial_2_0
       "tbz x11, #1, 45f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
       "tbz x11, #0, 46f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
       "b 46f\n"
       "45:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
       "46:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 48f\n"
       "47:"  // Height 2: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
       "48:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -511,98 +511,98 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "50:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "51:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 52f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 53f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 53f\n"
       "52:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "53:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 56f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
+      "cmp x27, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 55f\n"
       "54:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "cmp x26, #0x8\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "ldr q7, [x10, #0x10]\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
       "bge 54b\n"
       "55:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
+      "sub x27, x27, #0x4\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x10, x10, #0x80\n"
       "56:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 59f\n"
-      "cbz x26, 59f\n"
-      "tbz x26, #1, 57f\n"
-      "ldr d0, [x25], #0x8\n"
-      "ldr d1, [x24], #0x8\n"
-      "tbz x26, #0, 58f\n"
-      "ld1 { v0.s }[2], [x25]\n"
-      "ld1 { v1.s }[2], [x24]\n"
+      "cbz x27, 59f\n"
+      "cbz x27, 59f\n"
+      "tbz x27, #1, 57f\n"
+      "ldr d0, [x26], #0x8\n"
+      "ldr d1, [x25], #0x8\n"
+      "tbz x27, #0, 58f\n"
+      "ld1 { v0.s }[2], [x26]\n"
+      "ld1 { v1.s }[2], [x25]\n"
       "b 58f\n"
       "57:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
-      "ldr s1, [x24, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
+      "ldr s1, [x25, #0x0]\n"
       "58:"  // Height 2: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
@@ -616,148 +616,148 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
+      "add x10, x10, #0x80\n"
       "59:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 51b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 v6.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "tbz %x[flags], #1, 60f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v6.4s, v6.4s, v0.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v6.4s, v6.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v6.4s, v6.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmax v6.4s, v6.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
       "60:"  // Height 2: No activation
       "cmp x11, #0x10\n"
       "bge 69f\n"
       "tbz x11, #3, 64f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
       "tbz x11, #2, 62f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
       "tbz x11, #1, 61f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
       "b 68f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x11, #0, 68f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
       "b 68f\n"
       "62:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x11, #1, 63f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
       "b 68f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x11, #0, 68f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
       "b 68f\n"
       "64:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x11, #2, 66f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
       "tbz x11, #1, 65f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
       "b 68f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x11, #0, 68f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
       "b 68f\n"
       "66:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x11, #1, 67f\n"
-      "str d6, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
+      "str d6, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
       "tbz x11, #0, 68f\n"
-      "st1 { v6.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v6.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
       "b 68f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s6, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
+      "str s6, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
       "68:"  // Height 2: Partial direct writeback: Done
       "b 70f\n"
       "69:"  // Height 2: Full writeback
-      "str q6, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
+      "str q6, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
       "70:"  // Height 2: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 37b\n"
       "b 212f\n"
       "71:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "72:"  // Height 3: Column loop
-      "cbz x9, 73f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 73f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
@@ -767,111 +767,111 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "b 85f\n"
       "73:"  // Height 3: no bias
       "tbz %x[flags], #0, 84f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "bge 82f\n"
       "tbz x11, #3, 77f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #2, 75f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
       "tbz x11, #1, 74f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
       "b 81f\n"
       "74:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 81f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
       "b 81f\n"
       "75:"  // Height 3: Partial accumulate: partial_2_8
       "tbz x11, #1, 76f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
       "b 81f\n"
       "76:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 81f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
       "b 81f\n"
       "77:"  // Height 3: Partial accumulate: partial_4_0
       "tbz x11, #2, 79f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #1, 78f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
       "b 81f\n"
       "78:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 81f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
       "b 81f\n"
       "79:"  // Height 3: Partial accumulate: partial_2_0
       "tbz x11, #1, 80f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
       "tbz x11, #0, 81f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
       "b 81f\n"
       "80:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
       "81:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 83f\n"
       "82:"  // Height 3: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
       "83:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -908,96 +908,96 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "85:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "86:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 87f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 88f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 88f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 88f\n"
       "87:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "88:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 91f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
+      "cmp x27, #0x8\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 90f\n"
       "89:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "sub x27, x27, #0x4\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
       "add x10, x10, #0x80\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 89b\n"
       "90:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "sub x27, x27, #0x4\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
@@ -1009,32 +1009,32 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "91:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 94f\n"
-      "cbz x26, 94f\n"
-      "tbz x26, #1, 92f\n"
-      "ldr d0, [x25], #0x8\n"
-      "ldr d1, [x24], #0x8\n"
-      "ldr d2, [x23], #0x8\n"
-      "tbz x26, #0, 93f\n"
-      "ld1 { v0.s }[2], [x25]\n"
-      "ld1 { v1.s }[2], [x24]\n"
-      "ld1 { v2.s }[2], [x23]\n"
+      "cbz x27, 94f\n"
+      "cbz x27, 94f\n"
+      "tbz x27, #1, 92f\n"
+      "ldr d0, [x26], #0x8\n"
+      "ldr d1, [x25], #0x8\n"
+      "ldr d2, [x24], #0x8\n"
+      "tbz x27, #0, 93f\n"
+      "ld1 { v0.s }[2], [x26]\n"
+      "ld1 { v1.s }[2], [x25]\n"
+      "ld1 { v2.s }[2], [x24]\n"
       "b 93f\n"
       "92:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
-      "ldr s1, [x24, #0x0]\n"
-      "ldr s2, [x23, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
+      "ldr s1, [x25, #0x0]\n"
+      "ldr s2, [x24, #0x0]\n"
       "93:"  // Height 3: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
@@ -1054,21 +1054,21 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "94:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 86b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v6.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
@@ -1077,155 +1077,155 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "tbz %x[flags], #1, 95f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v6.4s, v6.4s, v0.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v6.4s, v6.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v6.4s, v6.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v6.4s, v6.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "95:"  // Height 3: No activation
       "cmp x11, #0x10\n"
       "bge 104f\n"
       "tbz x11, #3, 99f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
-      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
+      "st1 { v17.4s }, [x24], #0x10\n"
       "tbz x11, #2, 97f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v18.4s }, [x24], #0x10\n"
       "tbz x11, #1, 96f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d19, [x23], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d19, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v19.s }[2], [x24]\n"
       "b 103f\n"
       "96:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x11, #0, 103f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s19, [x23, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s19, [x24, #0x0]\n"
       "b 103f\n"
       "97:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x11, #1, 98f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d18, [x23], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d18, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v18.s }[2], [x24]\n"
       "b 103f\n"
       "98:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x11, #0, 103f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s18, [x23, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s18, [x24, #0x0]\n"
       "b 103f\n"
       "99:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x11, #2, 101f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v16.4s }, [x24], #0x10\n"
       "tbz x11, #1, 100f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d17, [x23], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d17, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v17.s }[2], [x24]\n"
       "b 103f\n"
       "100:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x11, #0, 103f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s17, [x23, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s17, [x24, #0x0]\n"
       "b 103f\n"
       "101:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x11, #1, 102f\n"
-      "str d6, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d16, [x23], #0x8\n"
+      "str d6, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d16, [x24], #0x8\n"
       "tbz x11, #0, 103f\n"
-      "st1 { v6.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v6.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v16.s }[2], [x24]\n"
       "b 103f\n"
       "102:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s6, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s16, [x23, #0x0]\n"
+      "str s6, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s16, [x24, #0x0]\n"
       "103:"  // Height 3: Partial direct writeback: Done
       "b 105f\n"
       "104:"  // Height 3: Full writeback
-      "str q6, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q16, [x23, #0x0]\n"
-      "str q17, [x23, #0x10]\n"
-      "str q18, [x23, #0x20]\n"
-      "str q19, [x23, #0x30]\n"
+      "str q6, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q16, [x24, #0x0]\n"
+      "str q17, [x24, #0x10]\n"
+      "str q18, [x24, #0x20]\n"
+      "str q19, [x24, #0x30]\n"
       "105:"  // Height 3: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 72b\n"
       "b 212f\n"
       "106:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "107:"  // Height 4: Column loop
-      "cbz x9, 108f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 108f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
@@ -1235,132 +1235,132 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "b 120f\n"
       "108:"  // Height 4: no bias
       "tbz %x[flags], #0, 119f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 117f\n"
       "tbz x11, #3, 112f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
       "tbz x11, #2, 110f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
       "tbz x11, #1, 109f\n"
-      "mov x19, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
       "b 116f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 116f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
       "b 116f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_8
       "tbz x11, #1, 111f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
       "b 116f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 116f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
       "b 116f\n"
       "112:"  // Height 4: Partial accumulate: partial_4_0
       "tbz x11, #2, 114f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
       "tbz x11, #1, 113f\n"
-      "mov x19, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
       "b 116f\n"
       "113:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 116f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
       "b 116f\n"
       "114:"  // Height 4: Partial accumulate: partial_2_0
       "tbz x11, #1, 115f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
       "tbz x11, #0, 116f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
       "b 116f\n"
       "115:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
       "116:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 118f\n"
       "117:"  // Height 4: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
       "118:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1397,65 +1397,67 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "120:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "121:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 122f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 123f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 123f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 123f\n"
       "122:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "123:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 126f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "cmp x27, #0x8\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 125f\n"
       "124:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "cmp x26, #0x8\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
-      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
@@ -1464,32 +1466,30 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 124b\n"
       "125:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "sub x27, x27, #0x4\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
@@ -1507,29 +1507,29 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "126:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 129f\n"
-      "cbz x26, 129f\n"
-      "tbz x26, #1, 127f\n"
-      "ldr d0, [x25], #0x8\n"
-      "ldr d1, [x24], #0x8\n"
-      "ldr d2, [x23], #0x8\n"
-      "ldr d3, [x22], #0x8\n"
-      "tbz x26, #0, 128f\n"
-      "ld1 { v0.s }[2], [x25]\n"
-      "ld1 { v1.s }[2], [x24]\n"
-      "ld1 { v2.s }[2], [x23]\n"
-      "ld1 { v3.s }[2], [x22]\n"
+      "cbz x27, 129f\n"
+      "cbz x27, 129f\n"
+      "tbz x27, #1, 127f\n"
+      "ldr d0, [x26], #0x8\n"
+      "ldr d1, [x25], #0x8\n"
+      "ldr d2, [x24], #0x8\n"
+      "ldr d3, [x23], #0x8\n"
+      "tbz x27, #0, 128f\n"
+      "ld1 { v0.s }[2], [x26]\n"
+      "ld1 { v1.s }[2], [x25]\n"
+      "ld1 { v2.s }[2], [x24]\n"
+      "ld1 { v3.s }[2], [x23]\n"
       "b 128f\n"
       "127:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
-      "ldr s1, [x24, #0x0]\n"
-      "ldr s2, [x23, #0x0]\n"
-      "ldr s3, [x22, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
+      "ldr s1, [x25, #0x0]\n"
+      "ldr s2, [x24, #0x0]\n"
+      "ldr s3, [x23, #0x0]\n"
       "128:"  // Height 4: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
@@ -1550,31 +1550,31 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec0e  // bfmmla v14.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       "129:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 121b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v6.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
@@ -1585,190 +1585,190 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "tbz %x[flags], #1, 130f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v6.4s, v6.4s, v0.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v6.4s, v6.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v6.4s, v6.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmax v6.4s, v6.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
       "130:"  // Height 4: No activation
       "cmp x11, #0x10\n"
       "bge 139f\n"
       "tbz x11, #3, 134f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
       "tbz x11, #2, 132f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
       "tbz x11, #1, 131f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 138f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x11, #0, 138f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 138f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x11, #1, 133f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 138f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x11, #0, 138f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 138f\n"
       "134:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x11, #2, 136f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
       "tbz x11, #1, 135f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 138f\n"
       "135:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x11, #0, 138f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 138f\n"
       "136:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x11, #1, 137f\n"
-      "str d6, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
+      "str d6, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
       "tbz x11, #0, 138f\n"
-      "st1 { v6.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v6.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 138f\n"
       "137:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s6, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s6, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "138:"  // Height 4: Partial direct writeback: Done
       "b 140f\n"
       "139:"  // Height 4: Full writeback
-      "str q6, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q6, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "140:"  // Height 4: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 107b\n"
       "b 212f\n"
       "141:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "142:"  // Height 5: Column loop
-      "cbz x9, 143f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 143f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
-      "mov v24.16b, v8.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v22.16b, v14.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v23.16b, v15.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v28.16b, v12.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v29.16b, v13.16b\n"
@@ -1779,153 +1779,153 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "b 155f\n"
       "143:"  // Height 5: no bias
       "tbz %x[flags], #0, 154f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 152f\n"
       "tbz x11, #3, 147f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #2, 145f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
       "tbz x11, #1, 144f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
       "b 151f\n"
       "144:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 151f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
       "b 151f\n"
       "145:"  // Height 5: Partial accumulate: partial_2_8
       "tbz x11, #1, 146f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
       "b 151f\n"
       "146:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 151f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
       "b 151f\n"
       "147:"  // Height 5: Partial accumulate: partial_4_0
       "tbz x11, #2, 149f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #1, 148f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
       "b 151f\n"
       "148:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 151f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
       "b 151f\n"
       "149:"  // Height 5: Partial accumulate: partial_2_0
       "tbz x11, #1, 150f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
       "tbz x11, #0, 151f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
       "b 151f\n"
       "150:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
       "151:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 153f\n"
       "152:"  // Height 5: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
       "153:"  // Height 5: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1978,72 +1978,74 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "155:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "156:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 157f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 158f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 158f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 158f\n"
       "157:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "158:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 161f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "cmp x27, #0x8\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
+      "ld1 { v4.4s }, [x22], #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 160f\n"
       "159:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "cmp x26, #0x8\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
-      "ld1 { v4.4s }, [x21], #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
-      "ldr q7, [x10, #0x10]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec9d  // bfmmla v29.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
@@ -2058,38 +2060,36 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
+      "ld1 { v4.4s }, [x22], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 159b\n"
       "160:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v4.4s }, [x21], #0x10\n"
-      "ldr q6, [x10, #0x0]\n"
+      "sub x27, x27, #0x4\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q7, [x10, #0x10]\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
@@ -2113,42 +2113,42 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "161:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 164f\n"
-      "cbz x26, 164f\n"
-      "tbz x26, #1, 162f\n"
-      "ldr d0, [x25], #0x8\n"
-      "ldr d1, [x24], #0x8\n"
-      "ldr d2, [x23], #0x8\n"
-      "ldr d3, [x22], #0x8\n"
-      "ldr d4, [x21], #0x8\n"
-      "tbz x26, #0, 163f\n"
-      "ld1 { v0.s }[2], [x25]\n"
-      "ld1 { v1.s }[2], [x24]\n"
-      "ld1 { v2.s }[2], [x23]\n"
-      "ld1 { v3.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
+      "cbz x27, 164f\n"
+      "cbz x27, 164f\n"
+      "tbz x27, #1, 162f\n"
+      "ldr d0, [x26], #0x8\n"
+      "ldr d1, [x25], #0x8\n"
+      "ldr d2, [x24], #0x8\n"
+      "ldr d3, [x23], #0x8\n"
+      "ldr d4, [x22], #0x8\n"
+      "tbz x27, #0, 163f\n"
+      "ld1 { v0.s }[2], [x26]\n"
+      "ld1 { v1.s }[2], [x25]\n"
+      "ld1 { v2.s }[2], [x24]\n"
+      "ld1 { v3.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
       "b 163f\n"
       "162:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
-      "ldr s1, [x24, #0x0]\n"
-      "ldr s2, [x23, #0x0]\n"
-      "ldr s3, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
+      "ldr s1, [x25, #0x0]\n"
+      "ldr s2, [x24, #0x0]\n"
+      "ldr s3, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
       "163:"  // Height 5: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q7, [x10, #0x10]\n"
-      ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
-      ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
+      ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
+      ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
@@ -2166,37 +2166,37 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec56  // bfmmla v22.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9e  // bfmmla v30.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x70]\n"
-      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "164:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 156b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v6.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
@@ -2209,221 +2209,221 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "uzp1 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "tbz %x[flags], #1, 165f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v6.4s, v6.4s, v0.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v6.4s, v6.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v6.4s, v6.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v6.4s, v6.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
       "165:"  // Height 5: No activation
       "cmp x11, #0x10\n"
       "bge 174f\n"
       "tbz x11, #3, 169f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
-      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
+      "st1 { v25.4s }, [x22], #0x10\n"
       "tbz x11, #2, 167f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v26.4s }, [x22], #0x10\n"
       "tbz x11, #1, 166f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d27, [x21], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d27, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v27.s }[2], [x22]\n"
       "b 173f\n"
       "166:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x11, #0, 173f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s27, [x21, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s27, [x22, #0x0]\n"
       "b 173f\n"
       "167:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x11, #1, 168f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d26, [x21], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d26, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v26.s }[2], [x22]\n"
       "b 173f\n"
       "168:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x11, #0, 173f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s26, [x21, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s26, [x22, #0x0]\n"
       "b 173f\n"
       "169:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x11, #2, 171f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v24.4s }, [x22], #0x10\n"
       "tbz x11, #1, 170f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d25, [x21], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d25, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v25.s }[2], [x22]\n"
       "b 173f\n"
       "170:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x11, #0, 173f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s25, [x21, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s25, [x22, #0x0]\n"
       "b 173f\n"
       "171:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x11, #1, 172f\n"
-      "str d6, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d24, [x21], #0x8\n"
+      "str d6, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d24, [x22], #0x8\n"
       "tbz x11, #0, 173f\n"
-      "st1 { v6.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v6.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v24.s }[2], [x22]\n"
       "b 173f\n"
       "172:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s6, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s24, [x21, #0x0]\n"
+      "str s6, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s24, [x22, #0x0]\n"
       "173:"  // Height 5: Partial direct writeback: Done
       "b 175f\n"
       "174:"  // Height 5: Full writeback
-      "str q6, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q24, [x21, #0x0]\n"
-      "str q25, [x21, #0x10]\n"
-      "str q26, [x21, #0x20]\n"
-      "str q27, [x21, #0x30]\n"
+      "str q6, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q24, [x22, #0x0]\n"
+      "str q25, [x22, #0x10]\n"
+      "str q26, [x22, #0x20]\n"
+      "str q27, [x22, #0x30]\n"
       "175:"  // Height 5: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 142b\n"
       "b 212f\n"
       "176:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "177:"  // Height 6: Column loop
-      "cbz x9, 178f\n"
-      "ldr q8, [x9, #0x0]\n"
+      "cbz x12, 178f\n"
+      "ldr q8, [x12, #0x0]\n"
+      "ldr q9, [x12, #0x10]\n"
       "zip2 v12.2d, v8.2d, v8.2d\n"
-      "ldr q9, [x9, #0x10]\n"
       "zip1 v8.2d, v8.2d, v8.2d\n"
-      "ldr q10, [x9, #0x20]\n"
-      "mov v16.16b, v8.16b\n"
-      "ldr q11, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "mov v20.16b, v12.16b\n"
-      "mov v24.16b, v8.16b\n"
+      "ldr q10, [x12, #0x20]\n"
+      "ldr q11, [x12, #0x30]\n"
       "zip2 v13.2d, v9.2d, v9.2d\n"
       "zip1 v9.2d, v9.2d, v9.2d\n"
       "zip2 v14.2d, v10.2d, v10.2d\n"
       "zip1 v10.2d, v10.2d, v10.2d\n"
+      "add x12, x12, #0x40\n"
       "zip2 v15.2d, v11.2d, v11.2d\n"
       "zip1 v11.2d, v11.2d, v11.2d\n"
+      "mov v16.16b, v8.16b\n"
+      "mov v20.16b, v12.16b\n"
       "mov v17.16b, v9.16b\n"
       "mov v21.16b, v13.16b\n"
       "mov v18.16b, v10.16b\n"
       "mov v22.16b, v14.16b\n"
       "mov v19.16b, v11.16b\n"
       "mov v23.16b, v15.16b\n"
+      "mov v24.16b, v8.16b\n"
       "mov v28.16b, v12.16b\n"
       "mov v25.16b, v9.16b\n"
       "mov v29.16b, v13.16b\n"
@@ -2434,174 +2434,174 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "b 190f\n"
       "178:"  // Height 6: no bias
       "tbz %x[flags], #0, 189f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "cmp x11, #0x10\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 187f\n"
       "tbz x11, #3, 182f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x24], #0x10\n"
-      "ld1 { v18.4s }, [x23], #0x10\n"
-      "ld1 { v21.4s }, [x22], #0x10\n"
-      "ld1 { v26.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x25], #0x10\n"
+      "ld1 { v18.4s }, [x24], #0x10\n"
+      "ld1 { v21.4s }, [x23], #0x10\n"
+      "ld1 { v26.4s }, [x22], #0x10\n"
+      "ld1 { v29.4s }, [x21], #0x10\n"
       "tbz x11, #2, 180f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x24], #0x10\n"
-      "ld1 { v19.4s }, [x23], #0x10\n"
-      "ld1 { v22.4s }, [x22], #0x10\n"
-      "ld1 { v27.4s }, [x21], #0x10\n"
-      "ld1 { v30.4s }, [x20], #0x10\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x25], #0x10\n"
+      "ld1 { v19.4s }, [x24], #0x10\n"
+      "ld1 { v22.4s }, [x23], #0x10\n"
+      "ld1 { v27.4s }, [x22], #0x10\n"
+      "ld1 { v30.4s }, [x21], #0x10\n"
       "tbz x11, #1, 179f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x19, #0x38\n"
-      "ldr d15, [x24], #0x8\n"
-      "ldr d24, [x23], #0x8\n"
-      "ldr d23, [x22], #0x8\n"
-      "ldr d6, [x21], #0x8\n"
-      "ldr d31, [x20], #0x8\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x25], #0x8\n"
+      "mov x20, #0x38\n"
+      "ldr d24, [x24], #0x8\n"
+      "ldr d23, [x23], #0x8\n"
+      "ldr d6, [x22], #0x8\n"
+      "ldr d31, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x24]\n"
-      "ld1 { v24.s }[2], [x23]\n"
-      "ld1 { v23.s }[2], [x22]\n"
-      "ld1 { v6.s }[2], [x21]\n"
-      "ld1 { v31.s }[2], [x20]\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x25]\n"
+      "ld1 { v24.s }[2], [x24]\n"
+      "ld1 { v23.s }[2], [x23]\n"
+      "ld1 { v6.s }[2], [x22]\n"
+      "ld1 { v31.s }[2], [x21]\n"
       "b 186f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x19, #0x30\n"
+      "mov x20, #0x30\n"
       "tbz x11, #0, 186f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x24, #0x0]\n"
-      "ldr s24, [x23, #0x0]\n"
-      "ldr s23, [x22, #0x0]\n"
-      "ldr s6, [x21, #0x0]\n"
-      "ldr s31, [x20, #0x0]\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x25, #0x0]\n"
+      "ldr s24, [x24, #0x0]\n"
+      "ldr s23, [x23, #0x0]\n"
+      "ldr s6, [x22, #0x0]\n"
+      "ldr s31, [x21, #0x0]\n"
       "b 186f\n"
       "180:"  // Height 6: Partial accumulate: partial_2_8
       "tbz x11, #1, 181f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x24], #0x8\n"
-      "mov x19, #0x28\n"
-      "ldr d19, [x23], #0x8\n"
-      "ldr d22, [x22], #0x8\n"
-      "ldr d27, [x21], #0x8\n"
-      "ldr d30, [x20], #0x8\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x25], #0x8\n"
+      "mov x20, #0x28\n"
+      "ldr d19, [x24], #0x8\n"
+      "ldr d22, [x23], #0x8\n"
+      "ldr d27, [x22], #0x8\n"
+      "ldr d30, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x24]\n"
-      "ld1 { v19.s }[2], [x23]\n"
-      "ld1 { v22.s }[2], [x22]\n"
-      "ld1 { v27.s }[2], [x21]\n"
-      "ld1 { v30.s }[2], [x20]\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x25]\n"
+      "ld1 { v19.s }[2], [x24]\n"
+      "ld1 { v22.s }[2], [x23]\n"
+      "ld1 { v27.s }[2], [x22]\n"
+      "ld1 { v30.s }[2], [x21]\n"
       "b 186f\n"
       "181:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x19, #0x20\n"
+      "mov x20, #0x20\n"
       "tbz x11, #0, 186f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x24, #0x0]\n"
-      "ldr s19, [x23, #0x0]\n"
-      "ldr s22, [x22, #0x0]\n"
-      "ldr s27, [x21, #0x0]\n"
-      "ldr s30, [x20, #0x0]\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x25, #0x0]\n"
+      "ldr s19, [x24, #0x0]\n"
+      "ldr s22, [x23, #0x0]\n"
+      "ldr s27, [x22, #0x0]\n"
+      "ldr s30, [x21, #0x0]\n"
       "b 186f\n"
       "182:"  // Height 6: Partial accumulate: partial_4_0
       "tbz x11, #2, 184f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x24], #0x10\n"
-      "ld1 { v17.4s }, [x23], #0x10\n"
-      "ld1 { v20.4s }, [x22], #0x10\n"
-      "ld1 { v25.4s }, [x21], #0x10\n"
-      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x25], #0x10\n"
+      "ld1 { v17.4s }, [x24], #0x10\n"
+      "ld1 { v20.4s }, [x23], #0x10\n"
+      "ld1 { v25.4s }, [x22], #0x10\n"
+      "ld1 { v28.4s }, [x21], #0x10\n"
       "tbz x11, #1, 183f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x19, #0x18\n"
-      "ldr d13, [x24], #0x8\n"
-      "ldr d18, [x23], #0x8\n"
-      "ldr d21, [x22], #0x8\n"
-      "ldr d26, [x21], #0x8\n"
-      "ldr d29, [x20], #0x8\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x25], #0x8\n"
+      "mov x20, #0x18\n"
+      "ldr d18, [x24], #0x8\n"
+      "ldr d21, [x23], #0x8\n"
+      "ldr d26, [x22], #0x8\n"
+      "ldr d29, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x24]\n"
-      "ld1 { v18.s }[2], [x23]\n"
-      "ld1 { v21.s }[2], [x22]\n"
-      "ld1 { v26.s }[2], [x21]\n"
-      "ld1 { v29.s }[2], [x20]\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x25]\n"
+      "ld1 { v18.s }[2], [x24]\n"
+      "ld1 { v21.s }[2], [x23]\n"
+      "ld1 { v26.s }[2], [x22]\n"
+      "ld1 { v29.s }[2], [x21]\n"
       "b 186f\n"
       "183:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x19, #0x10\n"
+      "mov x20, #0x10\n"
       "tbz x11, #0, 186f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x24, #0x0]\n"
-      "ldr s18, [x23, #0x0]\n"
-      "ldr s21, [x22, #0x0]\n"
-      "ldr s26, [x21, #0x0]\n"
-      "ldr s29, [x20, #0x0]\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x25, #0x0]\n"
+      "ldr s18, [x24, #0x0]\n"
+      "ldr s21, [x23, #0x0]\n"
+      "ldr s26, [x22, #0x0]\n"
+      "ldr s29, [x21, #0x0]\n"
       "b 186f\n"
       "184:"  // Height 6: Partial accumulate: partial_2_0
       "tbz x11, #1, 185f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x24], #0x8\n"
-      "mov x19, #0x8\n"
-      "ldr d17, [x23], #0x8\n"
-      "ldr d20, [x22], #0x8\n"
-      "ldr d25, [x21], #0x8\n"
-      "ldr d28, [x20], #0x8\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x25], #0x8\n"
+      "mov x20, #0x8\n"
+      "ldr d17, [x24], #0x8\n"
+      "ldr d20, [x23], #0x8\n"
+      "ldr d25, [x22], #0x8\n"
+      "ldr d28, [x21], #0x8\n"
       "tbz x11, #0, 186f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x24]\n"
-      "ld1 { v17.s }[2], [x23]\n"
-      "ld1 { v20.s }[2], [x22]\n"
-      "ld1 { v25.s }[2], [x21]\n"
-      "ld1 { v28.s }[2], [x20]\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x25]\n"
+      "ld1 { v17.s }[2], [x24]\n"
+      "ld1 { v20.s }[2], [x23]\n"
+      "ld1 { v25.s }[2], [x22]\n"
+      "ld1 { v28.s }[2], [x21]\n"
       "b 186f\n"
       "185:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x19, #0x0\n"
-      "ldr s12, [x24, #0x0]\n"
-      "ldr s17, [x23, #0x0]\n"
-      "ldr s20, [x22, #0x0]\n"
-      "ldr s25, [x21, #0x0]\n"
-      "ldr s28, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x25, #0x0]\n"
+      "mov x20, #0x0\n"
+      "ldr s17, [x24, #0x0]\n"
+      "ldr s20, [x23, #0x0]\n"
+      "ldr s25, [x22, #0x0]\n"
+      "ldr s28, [x21, #0x0]\n"
       "186:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x19\n"
+      "sub x9, x9, x20\n"
       "b 188f\n"
       "187:"  // Height 6: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x24, #0x0]\n"
-      "ldr q13, [x24, #0x10]\n"
-      "ldr q14, [x24, #0x20]\n"
-      "ldr q15, [x24, #0x30]\n"
-      "ldr q17, [x23, #0x0]\n"
-      "ldr q18, [x23, #0x10]\n"
-      "ldr q19, [x23, #0x20]\n"
-      "ldr q24, [x23, #0x30]\n"
-      "ldr q20, [x22, #0x0]\n"
-      "ldr q21, [x22, #0x10]\n"
-      "ldr q22, [x22, #0x20]\n"
-      "ldr q23, [x22, #0x30]\n"
-      "ldr q25, [x21, #0x0]\n"
-      "ldr q26, [x21, #0x10]\n"
-      "ldr q27, [x21, #0x20]\n"
-      "ldr q6, [x21, #0x30]\n"
-      "ldr q28, [x20, #0x0]\n"
-      "ldr q29, [x20, #0x10]\n"
-      "ldr q30, [x20, #0x20]\n"
-      "ldr q31, [x20, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x25, #0x0]\n"
+      "ldr q13, [x25, #0x10]\n"
+      "ldr q14, [x25, #0x20]\n"
+      "ldr q15, [x25, #0x30]\n"
+      "ldr q17, [x24, #0x0]\n"
+      "ldr q18, [x24, #0x10]\n"
+      "ldr q19, [x24, #0x20]\n"
+      "ldr q24, [x24, #0x30]\n"
+      "ldr q20, [x23, #0x0]\n"
+      "ldr q21, [x23, #0x10]\n"
+      "ldr q22, [x23, #0x20]\n"
+      "ldr q23, [x23, #0x30]\n"
+      "ldr q25, [x22, #0x0]\n"
+      "ldr q26, [x22, #0x10]\n"
+      "ldr q27, [x22, #0x20]\n"
+      "ldr q6, [x22, #0x30]\n"
+      "ldr q28, [x21, #0x0]\n"
+      "ldr q29, [x21, #0x10]\n"
+      "ldr q30, [x21, #0x20]\n"
+      "ldr q31, [x21, #0x30]\n"
       "188:"  // Height 6: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2654,80 +2654,83 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "190:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "191:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 192f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 193f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 193f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 193f\n"
       "192:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "193:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "blt 196f\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
-      "cmp x26, #0x8\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
+      "cmp x27, #0x8\n"
+      "ld1 { v4.4s }, [x22], #0x10\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
+      "ld1 { v5.4s }, [x21], #0x10\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 195f\n"
       "194:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "cmp x26, #0x8\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
-      "ld1 { v4.4s }, [x21], #0x10\n"
-      ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ld1 { v5.4s }, [x20], #0x10\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x8\n"
       ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
-      ".inst 0x4ea168a4  // bfcvtn2 v4.8h, v5.4s\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
+      ".inst 0x4ea168a4  // bfcvtn2 v4.8h, v5.4s\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "ld1 { v1.4s }, [x25], #0x10\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "ld1 { v3.4s }, [x23], #0x10\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e47ec55  // bfmmla v21.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9d  // bfmmla v29.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
+      "ld1 { v5.4s }, [x21], #0x10\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec52  // bfmmla v18.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9a  // bfmmla v26.4s, v4.8h, v6.8h\n"
@@ -2740,42 +2743,39 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e46ec0b  // bfmmla v11.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec53  // bfmmla v19.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec9b  // bfmmla v27.4s, v4.8h, v6.8h\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6e47ec0f  // bfmmla v15.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.4s }, [x25], #0x10\n"
+      "ld1 { v0.4s }, [x26], #0x10\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
+      "ld1 { v2.4s }, [x24], #0x10\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
+      "ld1 { v4.4s }, [x22], #0x10\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 194b\n"
       "195:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
-      "ld1 { v1.4s }, [x24], #0x10\n"
-      "sub x26, x26, #0x4\n"
-      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
-      "ld1 { v2.4s }, [x23], #0x10\n"
-      "ld1 { v3.4s }, [x22], #0x10\n"
       ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
-      "ld1 { v4.4s }, [x21], #0x10\n"
-      "ld1 { v5.4s }, [x20], #0x10\n"
-      ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
-      "ldr q6, [x10, #0x0]\n"
-      "ldr q7, [x10, #0x10]\n"
+      "sub x27, x27, #0x4\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
+      ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "prfm pldl1keep, [x25, #0x80]\n"
+      ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x4ea168a4  // bfcvtn2 v4.8h, v5.4s\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e47ec54  // bfmmla v20.4s, v2.8h, v7.8h\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e47ec9c  // bfmmla v28.4s, v4.8h, v7.8h\n"
       "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec51  // bfmmla v17.4s, v2.8h, v6.8h\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e46ec99  // bfmmla v25.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
@@ -2798,40 +2798,40 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "196:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 199f\n"
-      "cbz x26, 199f\n"
-      "tbz x26, #1, 197f\n"
-      "ldr d0, [x25], #0x8\n"
-      "ldr d1, [x24], #0x8\n"
-      "ldr d2, [x23], #0x8\n"
-      "ldr d3, [x22], #0x8\n"
-      "ldr d4, [x21], #0x8\n"
-      "ldr d5, [x20], #0x8\n"
-      "tbz x26, #0, 198f\n"
-      "ld1 { v0.s }[2], [x25]\n"
-      "ld1 { v1.s }[2], [x24]\n"
-      "ld1 { v2.s }[2], [x23]\n"
-      "ld1 { v3.s }[2], [x22]\n"
-      "ld1 { v4.s }[2], [x21]\n"
-      "ld1 { v5.s }[2], [x20]\n"
+      "cbz x27, 199f\n"
+      "cbz x27, 199f\n"
+      "tbz x27, #1, 197f\n"
+      "ldr d0, [x26], #0x8\n"
+      "ldr d1, [x25], #0x8\n"
+      "ldr d2, [x24], #0x8\n"
+      "ldr d3, [x23], #0x8\n"
+      "ldr d4, [x22], #0x8\n"
+      "ldr d5, [x21], #0x8\n"
+      "tbz x27, #0, 198f\n"
+      "ld1 { v0.s }[2], [x26]\n"
+      "ld1 { v1.s }[2], [x25]\n"
+      "ld1 { v2.s }[2], [x24]\n"
+      "ld1 { v3.s }[2], [x23]\n"
+      "ld1 { v4.s }[2], [x22]\n"
+      "ld1 { v5.s }[2], [x21]\n"
       "b 198f\n"
       "197:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr s0, [x25, #0x0]\n"
-      "ldr s1, [x24, #0x0]\n"
-      "ldr s2, [x23, #0x0]\n"
-      "ldr s3, [x22, #0x0]\n"
-      "ldr s4, [x21, #0x0]\n"
-      "ldr s5, [x20, #0x0]\n"
+      "ldr s0, [x26, #0x0]\n"
+      "ldr s1, [x25, #0x0]\n"
+      "ldr s2, [x24, #0x0]\n"
+      "ldr s3, [x23, #0x0]\n"
+      "ldr s4, [x22, #0x0]\n"
+      "ldr s5, [x21, #0x0]\n"
       "198:"  // Height 6: Multiply loop: Ragged operand read: Done
-      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
       "ldr q6, [x10, #0x0]\n"
-      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       "ldr q7, [x10, #0x10]\n"
+      ".inst 0x0ea16800  // bfcvtn v0.4h, v0.4s\n"
+      ".inst 0x0ea16842  // bfcvtn v2.4h, v2.4s\n"
       ".inst 0x0ea16884  // bfcvtn v4.4h, v4.4s\n"
       ".inst 0x4ea16820  // bfcvtn2 v0.8h, v1.4s\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x4ea16862  // bfcvtn2 v2.8h, v3.4s\n"
       ".inst 0x4ea168a4  // bfcvtn2 v4.8h, v5.4s\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       ".inst 0x6e46ec50  // bfmmla v16.4s, v2.8h, v6.8h\n"
       ".inst 0x6e46ec98  // bfmmla v24.4s, v4.8h, v6.8h\n"
       "ldr q6, [x10, #0x20]\n"
@@ -2863,33 +2863,33 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e47ec9f  // bfmmla v31.4s, v4.8h, v7.8h\n"
       "199:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 191b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 v6.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x24, #0x0]\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x25, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
@@ -2904,232 +2904,232 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 (
       "uzp1 v30.2d, v27.2d, v31.2d\n"
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "tbz %x[flags], #1, 200f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1r { v1.4s }, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1r { v0.4s }, [x19]\n"
-      "fmin v6.4s, v6.4s, v0.4s\n"
-      "fmin v12.4s, v12.4s, v0.4s\n"
-      "fmin v13.4s, v13.4s, v0.4s\n"
-      "fmin v14.4s, v14.4s, v0.4s\n"
-      "fmax v6.4s, v6.4s, v1.4s\n"
-      "fmax v12.4s, v12.4s, v1.4s\n"
-      "fmax v13.4s, v13.4s, v1.4s\n"
-      "fmax v14.4s, v14.4s, v1.4s\n"
-      "fmin v8.4s, v8.4s, v0.4s\n"
-      "fmin v9.4s, v9.4s, v0.4s\n"
-      "fmin v10.4s, v10.4s, v0.4s\n"
-      "fmax v8.4s, v8.4s, v1.4s\n"
-      "fmax v9.4s, v9.4s, v1.4s\n"
-      "fmax v10.4s, v10.4s, v1.4s\n"
-      "fmin v11.4s, v11.4s, v0.4s\n"
-      "fmin v15.4s, v15.4s, v0.4s\n"
-      "fmin v20.4s, v20.4s, v0.4s\n"
-      "fmax v11.4s, v11.4s, v1.4s\n"
-      "fmax v15.4s, v15.4s, v1.4s\n"
-      "fmax v20.4s, v20.4s, v1.4s\n"
-      "fmin v21.4s, v21.4s, v0.4s\n"
-      "fmin v22.4s, v22.4s, v0.4s\n"
-      "fmin v16.4s, v16.4s, v0.4s\n"
-      "fmax v21.4s, v21.4s, v1.4s\n"
-      "fmax v22.4s, v22.4s, v1.4s\n"
-      "fmax v16.4s, v16.4s, v1.4s\n"
-      "fmin v17.4s, v17.4s, v0.4s\n"
-      "fmin v18.4s, v18.4s, v0.4s\n"
-      "fmin v19.4s, v19.4s, v0.4s\n"
-      "fmax v17.4s, v17.4s, v1.4s\n"
-      "fmax v18.4s, v18.4s, v1.4s\n"
-      "fmax v19.4s, v19.4s, v1.4s\n"
-      "fmin v23.4s, v23.4s, v0.4s\n"
-      "fmin v28.4s, v28.4s, v0.4s\n"
-      "fmin v29.4s, v29.4s, v0.4s\n"
-      "fmax v23.4s, v23.4s, v1.4s\n"
-      "fmax v28.4s, v28.4s, v1.4s\n"
-      "fmax v29.4s, v29.4s, v1.4s\n"
-      "fmin v30.4s, v30.4s, v0.4s\n"
-      "fmin v24.4s, v24.4s, v0.4s\n"
-      "fmin v25.4s, v25.4s, v0.4s\n"
-      "fmax v30.4s, v30.4s, v1.4s\n"
-      "fmax v24.4s, v24.4s, v1.4s\n"
-      "fmax v25.4s, v25.4s, v1.4s\n"
-      "fmin v26.4s, v26.4s, v0.4s\n"
-      "fmin v27.4s, v27.4s, v0.4s\n"
-      "fmax v26.4s, v26.4s, v1.4s\n"
-      "fmax v27.4s, v27.4s, v1.4s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1r { v1.4s }, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1r { v0.4s }, [x20]\n"
+      "fmin v6.4s, v6.4s, v1.4s\n"
+      "fmin v12.4s, v12.4s, v1.4s\n"
+      "fmin v13.4s, v13.4s, v1.4s\n"
+      "fmin v14.4s, v14.4s, v1.4s\n"
+      "fmin v8.4s, v8.4s, v1.4s\n"
+      "fmin v9.4s, v9.4s, v1.4s\n"
+      "fmin v10.4s, v10.4s, v1.4s\n"
+      "fmin v11.4s, v11.4s, v1.4s\n"
+      "fmin v15.4s, v15.4s, v1.4s\n"
+      "fmin v20.4s, v20.4s, v1.4s\n"
+      "fmin v21.4s, v21.4s, v1.4s\n"
+      "fmin v22.4s, v22.4s, v1.4s\n"
+      "fmin v16.4s, v16.4s, v1.4s\n"
+      "fmin v17.4s, v17.4s, v1.4s\n"
+      "fmin v18.4s, v18.4s, v1.4s\n"
+      "fmin v19.4s, v19.4s, v1.4s\n"
+      "fmin v23.4s, v23.4s, v1.4s\n"
+      "fmin v28.4s, v28.4s, v1.4s\n"
+      "fmin v29.4s, v29.4s, v1.4s\n"
+      "fmin v30.4s, v30.4s, v1.4s\n"
+      "fmin v24.4s, v24.4s, v1.4s\n"
+      "fmin v25.4s, v25.4s, v1.4s\n"
+      "fmin v26.4s, v26.4s, v1.4s\n"
+      "fmin v27.4s, v27.4s, v1.4s\n"
+      "fmax v6.4s, v6.4s, v0.4s\n"
+      "fmax v12.4s, v12.4s, v0.4s\n"
+      "fmax v13.4s, v13.4s, v0.4s\n"
+      "fmax v14.4s, v14.4s, v0.4s\n"
+      "fmax v8.4s, v8.4s, v0.4s\n"
+      "fmax v9.4s, v9.4s, v0.4s\n"
+      "fmax v10.4s, v10.4s, v0.4s\n"
+      "fmax v11.4s, v11.4s, v0.4s\n"
+      "fmax v15.4s, v15.4s, v0.4s\n"
+      "fmax v20.4s, v20.4s, v0.4s\n"
+      "fmax v21.4s, v21.4s, v0.4s\n"
+      "fmax v22.4s, v22.4s, v0.4s\n"
+      "fmax v16.4s, v16.4s, v0.4s\n"
+      "fmax v17.4s, v17.4s, v0.4s\n"
+      "fmax v18.4s, v18.4s, v0.4s\n"
+      "fmax v19.4s, v19.4s, v0.4s\n"
+      "fmax v23.4s, v23.4s, v0.4s\n"
+      "fmax v28.4s, v28.4s, v0.4s\n"
+      "fmax v29.4s, v29.4s, v0.4s\n"
+      "fmax v30.4s, v30.4s, v0.4s\n"
+      "fmax v24.4s, v24.4s, v0.4s\n"
+      "fmax v25.4s, v25.4s, v0.4s\n"
+      "fmax v26.4s, v26.4s, v0.4s\n"
+      "fmax v27.4s, v27.4s, v0.4s\n"
       "200:"  // Height 6: No activation
       "cmp x11, #0x10\n"
       "bge 209f\n"
       "tbz x11, #3, 204f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v9.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v20.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v28.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v9.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v20.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v28.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
       "tbz x11, #2, 202f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x24], #0x10\n"
-      "st1 { v21.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v29.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x25], #0x10\n"
+      "st1 { v21.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v29.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
       "tbz x11, #1, 201f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x24], #0x8\n"
-      "str d22, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d30, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x25], #0x8\n"
+      "str d22, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d30, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x24]\n"
-      "st1 { v22.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v30.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x25]\n"
+      "st1 { v22.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v30.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 208f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x11, #0, 208f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x24, #0x0]\n"
-      "str s22, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s30, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x25, #0x0]\n"
+      "str s22, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s30, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 208f\n"
       "202:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x11, #1, 203f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x24], #0x8\n"
-      "str d21, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d29, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x25], #0x8\n"
+      "str d21, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d29, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x24]\n"
-      "st1 { v21.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v29.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x25]\n"
+      "st1 { v21.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v29.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 208f\n"
       "203:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x11, #0, 208f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x24, #0x0]\n"
-      "str s21, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s29, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x25, #0x0]\n"
+      "str s21, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s29, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 208f\n"
       "204:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x11, #2, 206f\n"
-      "st1 { v6.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x24], #0x10\n"
-      "st1 { v15.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v23.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
+      "st1 { v6.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x25], #0x10\n"
+      "st1 { v15.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v23.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
       "tbz x11, #1, 205f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x24], #0x8\n"
-      "str d20, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d28, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x25], #0x8\n"
+      "str d20, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d28, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x24]\n"
-      "st1 { v20.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v28.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x25]\n"
+      "st1 { v20.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v28.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 208f\n"
       "205:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x11, #0, 208f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x24, #0x0]\n"
-      "str s20, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s28, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x25, #0x0]\n"
+      "str s20, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s28, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 208f\n"
       "206:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x11, #1, 207f\n"
-      "str d6, [x28], #0x8\n"
-      "str d8, [x24], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d6, [x9], #0x8\n"
+      "str d8, [x25], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x11, #0, 208f\n"
-      "st1 { v6.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x24]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "st1 { v6.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x25]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 208f\n"
       "207:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s6, [x28, #0x0]\n"
-      "str s8, [x24, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s6, [x9, #0x0]\n"
+      "str s8, [x25, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "208:"  // Height 6: Partial direct writeback: Done
       "b 210f\n"
       "209:"  // Height 6: Full writeback
-      "str q6, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x24, #0x0]\n"
-      "str q9, [x24, #0x10]\n"
-      "str q10, [x24, #0x20]\n"
-      "str q11, [x24, #0x30]\n"
-      "str q15, [x23, #0x0]\n"
-      "str q20, [x23, #0x10]\n"
-      "str q21, [x23, #0x20]\n"
-      "str q22, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q23, [x21, #0x0]\n"
-      "str q28, [x21, #0x10]\n"
-      "str q29, [x21, #0x20]\n"
-      "str q30, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q6, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x25, #0x0]\n"
+      "str q9, [x25, #0x10]\n"
+      "str q10, [x25, #0x20]\n"
+      "str q11, [x25, #0x30]\n"
+      "str q15, [x24, #0x0]\n"
+      "str q20, [x24, #0x10]\n"
+      "str q21, [x24, #0x20]\n"
+      "str q22, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q23, [x22, #0x0]\n"
+      "str q28, [x22, #0x10]\n"
+      "str q29, [x22, #0x20]\n"
+      "str q30, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "210:"  // Height 6: Writeback done
       "subs x11, x11, #0x10\n"
       "bgt 177b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 212f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 211f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "211:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "212:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp
index ee7e55f179..b31b80586c 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,232 +85,220 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 61f\n"
       "beq 31f\n"
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
       "movi v15.16b, #0x1\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x14, %x[output_ptr]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "cbnz x12, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x10, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 11f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
-      "ldr x24, [x10, #0x18]\n"
-      "add x25, x25, #0x10\n"
-      "ldr d6, [x10, #0x20]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
+      "ldr d4, [x13, #0x70]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "mov v7.d[1], x19\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
-      "ldr x19, [x10, #0x58]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr d7, [x13, #0xa0]\n"
+      "mov v4.d[1], x9\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v9.d[1], x19\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
-      "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
-      "ldr d6, [x10, #0x90]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d4, [x13, #0xe0]\n"
+      "mov v5.d[1], x28\n"
+      "ldr x27, [x13, #0x98]\n"
+      "mov v6.d[1], x27\n"
+      "ldr x26, [x13, #0xa8]\n"
+      "mov v7.d[1], x26\n"
+      "ldr x25, [x13, #0xb8]\n"
+      "mov v8.d[1], x25\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
-      "mov v6.d[1], x23\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
-      "ldr x23, [x10, #0xb8]\n"
-      "mov v7.d[1], x19\n"
-      "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
-      "ldr x23, [x10, #0xd8]\n"
+      "ldr x28, [x13, #0xf8]\n"
+      "mov v9.d[1], x24\n"
+      "mov v10.d[1], x20\n"
+      "add x10, x10, #0x10\n"
+      "mov v4.d[1], x9\n"
+      "add x13, x13, #0x100\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr d4, [x10, #0xe0]\n"
-      "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
-      "ldr d5, [x10, #0xf0]\n"
-      "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e951  // sdot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x4fa0e892  // sdot v18.4s, v4.16b, v0.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x4fa0e8b3  // sdot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 8f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      "ldr q0, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q4, [x13, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
       "bge 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
-      "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x10\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q4, [x10, #0x70]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "ldr q5, [x10, #0x80]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x4fa0e951  // sdot v17.4s, v10.16b, v0.4b[3]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e892  // sdot v18.4s, v4.16b, v0.4b[3]\n"
       ".inst 0x4fa0e8b3  // sdot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 10f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 18f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 18f\n"
+      "cmp x11, #0x4\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f80e133  // sdot v19.4s, v9.16b, v0.4b[0]\n"
       "bge 12b\n"
-      "cbz x26, 18f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 15f\n"
-      "ldr h0, [x25], #0x2\n"
-      "tbz x26, #0, 16f\n"
-      "ld1 { v0.b }[2], [x25]\n"
+      "cbz x11, 18f\n"
+      "tbz x11, #1, 15f\n"
+      "ldr h0, [x10], #0x2\n"
+      "tbz x11, #0, 16f\n"
+      "ld1 { v0.b }[2], [x10]\n"
       "b 16f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 17f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "17:"  // Height 1: Multiply loop: unique 4: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x4f80e0d3  // sdot v19.4s, v6.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       "18:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "tbnz %x[flags], #31, 19f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v1.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "neg v1.4s, v1.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "19:"  // Height 1: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "ldr q3, [x9, #0x30]\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
       "add v18.4s, v18.4s, v2.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v19.4s, v19.4s, v3.4s\n"
-      "add x9, x9, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 20f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
@@ -329,90 +317,90 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "srshl v17.4s, v17.4s, v0.4s\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 29f\n"
-      "tbz x11, #3, 24f\n"
-      "str d16, [x28], #0x8\n"
-      "tbz x11, #2, 22f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "tbz x11, #1, 21f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[14], [x28]\n"
+      "tbz x15, #3, 24f\n"
+      "str d16, [x14], #0x8\n"
+      "tbz x15, #2, 22f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "tbz x15, #1, 21f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[14], [x14]\n"
       "b 28f\n"
       "21:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[12], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[12], [x14]\n"
       "b 28f\n"
       "22:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 23f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[10], [x28]\n"
+      "tbz x15, #1, 23f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[10], [x14]\n"
       "b 28f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[8], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[8], [x14]\n"
       "b 28f\n"
       "24:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 26f\n"
-      "str s16, [x28], #0x4\n"
-      "tbz x11, #1, 25f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[6], [x28]\n"
+      "tbz x15, #2, 26f\n"
+      "str s16, [x14], #0x4\n"
+      "tbz x15, #1, 25f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[6], [x14]\n"
       "b 28f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[4], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[4], [x14]\n"
       "b 28f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 27f\n"
-      "str h16, [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[2], [x28]\n"
+      "tbz x15, #1, 27f\n"
+      "str h16, [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[2], [x14]\n"
       "b 28f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
       "28:"  // Height 1: Partial direct writeback: Done
       "b 30f\n"
       "29:"  // Height 1: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
       "30:"  // Height 1: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 2b\n"
       "b 122f\n"
       "31:"  // Height 2
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "movi v15.16b, #0x1\n"
-      "mov x9, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "movi v15.16b, #0x1\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x14, %x[output_ptr]\n"
       "32:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -423,110 +411,98 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "33:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "34:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 35f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x27, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x12, 36f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
       "b 36f\n"
       "35:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
       "36:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 41f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 39f\n"
       "37:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
-      "ldr d6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
-      "ldr x23, [x10, #0x28]\n"
-      "add x22, x22, #0x10\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr d7, [x13, #0xa0]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr d8, [x13, #0xb0]\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
-      "ldr d6, [x10, #0x90]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d4, [x13, #0xe0]\n"
+      "mov v7.d[1], x26\n"
+      "ldr x24, [x13, #0xc8]\n"
+      "mov v8.d[1], x25\n"
+      "ldr x20, [x13, #0xd8]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
-      "mov v6.d[1], x23\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "ldr d5, [x13, #0xf0]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xb8]\n"
-      "mov v7.d[1], x19\n"
-      "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v8.d[1], x23\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x4f81e8f6  // sdot v22.4s, v7.16b, v1.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4f81e917  // sdot v23.4s, v8.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
+      "add x10, x10, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x4fa1e934  // sdot v20.4s, v9.16b, v1.4b[3]\n"
-      "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
-      "ldr d5, [x10, #0xf0]\n"
-      "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e951  // sdot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x4fa1e955  // sdot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x4fa0e892  // sdot v18.4s, v4.16b, v0.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x4fa1e896  // sdot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x4fa0e8b3  // sdot v19.4s, v5.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8b7  // sdot v23.4s, v5.16b, v1.4b[3]\n"
@@ -534,53 +510,53 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "38:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "bge 37b\n"
       "39:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8f6  // sdot v22.4s, v7.16b, v1.4b[2]\n"
@@ -598,143 +574,143 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 6: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "41:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 48f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 48f\n"
+      "cmp x11, #0x4\n"
       "blt 44f\n"
       "42:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
       "tbnz %x[flags], #31, 43f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "43:"  // Height 2: Multiply loop: unique 7: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x4f81e116  // sdot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x4f80e133  // sdot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x4f81e137  // sdot v23.4s, v9.16b, v1.4b[0]\n"
       "bge 42b\n"
-      "cbz x26, 48f\n"
       "44:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 45f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "tbz x26, #0, 46f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
+      "cbz x11, 48f\n"
+      "tbz x11, #1, 45f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "tbz x11, #0, 46f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
       "b 46f\n"
       "45:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
       "46:"  // Height 2: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 47f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "47:"  // Height 2: Multiply loop: unique 8: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f81e0b6  // sdot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x4f80e0d3  // sdot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0d7  // sdot v23.4s, v6.16b, v1.4b[0]\n"
       "48:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 34b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "tbnz %x[flags], #31, 49f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "neg v2.4s, v2.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "49:"  // Height 2: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
-      "add v22.4s, v22.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
+      "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 50f\n"
       "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
+      "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
-      "and v7.16b, v19.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "and v4.16b, v23.16b, v0.16b\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "50:"  // Height 2: no shift correction
       "srshl v16.4s, v16.4s, v0.4s\n"
@@ -745,13 +721,8 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "srshl v21.4s, v21.4s, v0.4s\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -760,6 +731,8 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "add v21.4s, v21.4s, v4.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -768,6 +741,8 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -780,87 +755,88 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 59f\n"
-      "tbz x11, #3, 54f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x11, #2, 52f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "tbz x11, #1, 51f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "tbz x15, #3, 54f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x15, #2, 52f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "tbz x15, #1, 51f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 58f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 58f\n"
       "52:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 53f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "tbz x15, #1, 53f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 58f\n"
       "53:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 58f\n"
       "54:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 56f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "tbz x11, #1, 55f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "tbz x15, #2, 56f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "tbz x15, #1, 55f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 58f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 58f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 57f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "tbz x15, #1, 57f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 58f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "58:"  // Height 2: Partial direct writeback: Done
       "b 60f\n"
       "59:"  // Height 2: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
       "60:"  // Height 2: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 32b\n"
       "b 122f\n"
       "61:"  // Height 3
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x9, %x[col_bias]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x14, %x[output_ptr]\n"
       "62:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -875,129 +851,117 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "63:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "64:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 65f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x27, 66f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x12, 66f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 66f\n"
       "65:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
-      "add x21, x22, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
+      "add x22, x23, x20\n"
       "66:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 71f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 69f\n"
       "67:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr d6, [x10, #0x20]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "add x25, x25, #0x10\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
-      "add x22, x22, #0x10\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "mov v8.d[1], x23\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr d10, [x10, #0x60]\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x4f82e0fb  // sdot v27.4s, v7.16b, v2.4b[0]\n"
-      "mov v9.d[1], x19\n"
+      "ldr d7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "mov v7.d[1], x26\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x4fa2e118  // sdot v24.4s, v8.16b, v2.4b[1]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "mov v8.d[1], x25\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x4fa2e139  // sdot v25.4s, v9.16b, v2.4b[1]\n"
-      "mov v4.d[1], x19\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr d6, [x10, #0x90]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x4fa2e15a  // sdot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x4fa2e09b  // sdot v27.4s, v4.16b, v2.4b[1]\n"
-      "mov v6.d[1], x23\n"
+      "ldr d4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
-      "ldr x23, [x10, #0xb8]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "mov v7.d[1], x19\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr d9, [x10, #0xc0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
-      "mov v8.d[1], x23\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4f81e8f6  // sdot v22.4s, v7.16b, v1.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
       ".inst 0x4f82e8fa  // sdot v26.4s, v7.16b, v2.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x4f81e917  // sdot v23.4s, v8.16b, v1.4b[2]\n"
-      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x4f82e91b  // sdot v27.4s, v8.16b, v2.4b[2]\n"
-      "mov v10.d[1], x23\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr x19, [x10, #0xe8]\n"
       ".inst 0x4fa1e934  // sdot v20.4s, v9.16b, v1.4b[3]\n"
-      "ldr d5, [x10, #0xf0]\n"
       ".inst 0x4fa2e938  // sdot v24.4s, v9.16b, v2.4b[3]\n"
-      "ldr x24, [x10, #0xf8]\n"
       ".inst 0x4fa0e951  // sdot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x4fa1e955  // sdot v21.4s, v10.16b, v1.4b[3]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4fa2e959  // sdot v25.4s, v10.16b, v2.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x4fa0e892  // sdot v18.4s, v4.16b, v0.4b[3]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x4fa1e896  // sdot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x4fa2e89a  // sdot v26.4s, v4.16b, v2.4b[3]\n"
       ".inst 0x4fa0e8b3  // sdot v19.4s, v5.16b, v0.4b[3]\n"
@@ -1008,65 +972,65 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "68:"  // Height 3: Multiply loop: unique 9: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
       "bge 67b\n"
       "69:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f82e0fb  // sdot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4fa2e118  // sdot v24.4s, v8.16b, v2.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x4fa2e139  // sdot v25.4s, v9.16b, v2.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x4fa2e15a  // sdot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x4fa2e09b  // sdot v27.4s, v4.16b, v2.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
@@ -1092,32 +1056,32 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "70:"  // Height 3: Multiply loop: unique 10: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "71:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 78f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 78f\n"
+      "cmp x11, #0x4\n"
       "blt 74f\n"
       "72:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
       "tbnz %x[flags], #31, 73f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "73:"  // Height 3: Multiply loop: unique 11: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f9  // sdot v25.4s, v7.16b, v2.4b[0]\n"
@@ -1128,37 +1092,37 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4f81e137  // sdot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x4f82e13b  // sdot v27.4s, v9.16b, v2.4b[0]\n"
       "bge 72b\n"
-      "cbz x26, 78f\n"
       "74:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 75f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "tbz x26, #0, 76f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
+      "cbz x11, 78f\n"
+      "tbz x11, #1, 75f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "tbz x11, #0, 76f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
       "b 76f\n"
       "75:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
       "76:"  // Height 3: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 12: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f82e099  // sdot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x4f81e0b6  // sdot v22.4s, v5.16b, v1.4b[0]\n"
@@ -1167,22 +1131,22 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4f81e0d7  // sdot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0db  // sdot v27.4s, v6.16b, v2.4b[0]\n"
       "78:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 64b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbnz %x[flags], #31, 79f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "neg v3.4s, v3.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
@@ -1191,9 +1155,13 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "79:"  // Height 3: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
@@ -1203,73 +1171,69 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "add v25.4s, v25.4s, v13.4s\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
+      "add v23.4s, v23.4s, v3.4s\n"
       "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add v19.4s, v19.4s, v3.4s\n"
-      "add v23.4s, v23.4s, v3.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "sqrdmulh v24.4s, v24.4s, v4.4s\n"
       "sqrdmulh v25.4s, v25.4s, v4.4s\n"
       "sqrdmulh v26.4s, v26.4s, v4.4s\n"
       "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 80f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
-      "and v9.16b, v21.16b, v0.16b\n"
-      "and v10.16b, v22.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
-      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "and v9.16b, v21.16b, v0.16b\n"
+      "and v10.16b, v22.16b, v0.16b\n"
+      "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
       "and v6.16b, v25.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
@@ -1287,13 +1251,8 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "srshl v25.4s, v25.4s, v0.4s\n"
       "srshl v26.4s, v26.4s, v0.4s\n"
       "srshl v27.4s, v27.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -1304,6 +1263,10 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -1314,6 +1277,10 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -1324,120 +1291,117 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
+      "smax v26.4s, v26.4s, v5.4s\n"
+      "smax v27.4s, v27.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
+      "uzp1 v25.8h, v26.8h, v27.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
-      "smax v26.4s, v26.4s, v5.4s\n"
-      "smax v27.4s, v27.4s, v5.4s\n"
-      "uzp1 v25.8h, v26.8h, v27.8h\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 89f\n"
-      "tbz x11, #3, 84f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x11, #2, 82f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "tbz x11, #1, 81f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "tbz x15, #3, 84f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x15, #2, 82f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "tbz x15, #1, 81f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 88f\n"
       "81:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 88f\n"
       "82:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 83f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "tbz x15, #1, 83f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 88f\n"
       "83:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 88f\n"
       "84:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 86f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "tbz x11, #1, 85f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "tbz x15, #2, 86f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "tbz x15, #1, 85f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 88f\n"
       "85:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 88f\n"
       "86:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 87f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "tbz x15, #1, 87f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 88f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "88:"  // Height 3: Partial direct writeback: Done
       "b 90f\n"
       "89:"  // Height 3: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "90:"  // Height 3: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 62b\n"
       "b 122f\n"
       "91:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x9, %x[col_bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "movi v15.16b, #0x1\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
-      "mov x19, #0x4\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "92:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1456,137 +1420,125 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "93:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "94:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 95f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x27, 96f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x12, 96f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 96f\n"
       "95:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "96:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 101f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 99f\n"
       "97:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr d6, [x10, #0x20]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x4f83e09c  // sdot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
-      "add x25, x25, #0x10\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x4f83e0bd  // sdot v29.4s, v5.16b, v3.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
-      "mov v8.d[1], x23\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x4f83e0de  // sdot v30.4s, v6.16b, v3.4b[0]\n"
-      "ldr d10, [x10, #0x60]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x4f82e0fb  // sdot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x4f83e0ff  // sdot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "mov v7.d[1], x26\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4fa2e118  // sdot v24.4s, v8.16b, v2.4b[1]\n"
-      "mov v4.d[1], x19\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4fa3e11c  // sdot v28.4s, v8.16b, v3.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      "ldr d6, [x10, #0x90]\n"
+      "mov v8.d[1], x25\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4fa2e139  // sdot v25.4s, v9.16b, v2.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4fa3e13d  // sdot v29.4s, v9.16b, v3.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      "mov v6.d[1], x23\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr x19, [x10, #0xa8]\n"
       ".inst 0x4fa2e15a  // sdot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x4fa3e15e  // sdot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr x23, [x10, #0xb8]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v7.d[1], x19\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d9, [x10, #0xc0]\n"
       ".inst 0x4fa2e09b  // sdot v27.4s, v4.16b, v2.4b[1]\n"
-      "mov v8.d[1], x23\n"
       ".inst 0x4fa3e09f  // sdot v31.4s, v4.16b, v3.4b[1]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "ldr d4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x4f83e8bc  // sdot v28.4s, v5.16b, v3.4b[2]\n"
-      "ldr d4, [x10, #0xe0]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "mov v10.d[1], x23\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr x19, [x10, #0xe8]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
-      "ldr d5, [x10, #0xf0]\n"
       ".inst 0x4f83e8dd  // sdot v29.4s, v6.16b, v3.4b[2]\n"
-      "ldr x24, [x10, #0xf8]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x4f81e8f6  // sdot v22.4s, v7.16b, v1.4b[2]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f82e8fa  // sdot v26.4s, v7.16b, v2.4b[2]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x4f83e8fe  // sdot v30.4s, v7.16b, v3.4b[2]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4f81e917  // sdot v23.4s, v8.16b, v1.4b[2]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x4f82e91b  // sdot v27.4s, v8.16b, v2.4b[2]\n"
       ".inst 0x4f83e91f  // sdot v31.4s, v8.16b, v3.4b[2]\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
@@ -1611,77 +1563,77 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "98:"  // Height 4: Multiply loop: unique 13: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
       "bge 97b\n"
       "99:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f83e09c  // sdot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
       ".inst 0x4f83e0bd  // sdot v29.4s, v5.16b, v3.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f83e0de  // sdot v30.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4f82e0fb  // sdot v27.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0ff  // sdot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
       ".inst 0x4fa2e118  // sdot v24.4s, v8.16b, v2.4b[1]\n"
       ".inst 0x4fa3e11c  // sdot v28.4s, v8.16b, v3.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x4fa2e139  // sdot v25.4s, v9.16b, v2.4b[1]\n"
       ".inst 0x4fa3e13d  // sdot v29.4s, v9.16b, v3.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x4fa1e156  // sdot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x4fa2e15a  // sdot v26.4s, v10.16b, v2.4b[1]\n"
       ".inst 0x4fa3e15e  // sdot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x4fa1e097  // sdot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x4fa2e09b  // sdot v27.4s, v4.16b, v2.4b[1]\n"
       ".inst 0x4fa3e09f  // sdot v31.4s, v4.16b, v3.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
       ".inst 0x4f83e8bc  // sdot v28.4s, v5.16b, v3.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8dd  // sdot v29.4s, v6.16b, v3.4b[2]\n"
@@ -1715,35 +1667,35 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "100:"  // Height 4: Multiply loop: unique 14: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "101:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 108f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 108f\n"
+      "cmp x11, #0x4\n"
       "blt 104f\n"
       "102:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "tbnz %x[flags], #31, 103f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "103:"  // Height 4: Multiply loop: unique 15: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f83e0dc  // sdot v28.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
@@ -1758,24 +1710,24 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4f82e13b  // sdot v27.4s, v9.16b, v2.4b[0]\n"
       ".inst 0x4f83e13f  // sdot v31.4s, v9.16b, v3.4b[0]\n"
       "bge 102b\n"
-      "cbz x26, 108f\n"
       "104:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 105f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "ldr h3, [x20], #0x2\n"
-      "tbz x26, #0, 106f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
-      "ld1 { v3.b }[2], [x20]\n"
+      "cbz x11, 108f\n"
+      "tbz x11, #1, 105f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "ldr h3, [x21], #0x2\n"
+      "tbz x11, #0, 106f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
+      "ld1 { v3.b }[2], [x21]\n"
       "b 106f\n"
       "105:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
-      "ldr b3, [x20, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
+      "ldr b3, [x21, #0x0]\n"
       "106:"  // Height 4: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 107f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
@@ -1783,16 +1735,16 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "107:"  // Height 4: Multiply loop: unique 16: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x4f83e15c  // sdot v28.4s, v10.16b, v3.4b[0]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x4f82e099  // sdot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x4f83e09d  // sdot v29.4s, v4.16b, v3.4b[0]\n"
@@ -1805,28 +1757,28 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       ".inst 0x4f82e0db  // sdot v27.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0df  // sdot v31.4s, v6.16b, v3.4b[0]\n"
       "108:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 94b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "tbnz %x[flags], #31, 109f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "neg v4.4s, v4.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "neg v4.4s, v4.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
       "mul v11.4s, v11.4s, v4.4s\n"
@@ -1834,9 +1786,13 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "109:"  // Height 4: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
@@ -1850,39 +1806,34 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "add v29.4s, v29.4s, v14.4s\n"
       "add v30.4s, v30.4s, v14.4s\n"
       "add v31.4s, v31.4s, v14.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
+      "add v23.4s, v23.4s, v3.4s\n"
       "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
+      "add v27.4s, v27.4s, v3.4s\n"
       "add v28.4s, v28.4s, v0.4s\n"
       "add v29.4s, v29.4s, v1.4s\n"
       "add v30.4s, v30.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add v19.4s, v19.4s, v3.4s\n"
-      "add v23.4s, v23.4s, v3.4s\n"
-      "add v27.4s, v27.4s, v3.4s\n"
       "add v31.4s, v31.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "sqrdmulh v24.4s, v24.4s, v4.4s\n"
       "sqrdmulh v25.4s, v25.4s, v4.4s\n"
@@ -1892,53 +1843,54 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "sqrdmulh v29.4s, v29.4s, v4.4s\n"
       "sqrdmulh v30.4s, v30.4s, v4.4s\n"
       "sqrdmulh v31.4s, v31.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 110f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "sqadd v17.4s, v17.4s, v5.4s\n"
       "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
+      "and v4.16b, v23.16b, v0.16b\n"
+      "and v5.16b, v24.16b, v0.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sqadd v17.4s, v17.4s, v5.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
-      "and v5.16b, v24.16b, v0.16b\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
       "and v9.16b, v28.16b, v0.16b\n"
       "and v10.16b, v29.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
+      "and v4.16b, v30.16b, v0.16b\n"
+      "and v5.16b, v31.16b, v0.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "and v4.16b, v30.16b, v0.16b\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
-      "and v5.16b, v31.16b, v0.16b\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "sqadd v28.4s, v28.4s, v9.4s\n"
       "sqadd v29.4s, v29.4s, v10.4s\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v30.4s, v30.4s, v4.4s\n"
       "sqadd v31.4s, v31.4s, v5.4s\n"
       "110:"  // Height 4: no shift correction
@@ -1958,13 +1910,8 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "srshl v29.4s, v29.4s, v0.4s\n"
       "srshl v30.4s, v30.4s, v0.4s\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -1975,6 +1922,14 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -1985,6 +1940,14 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -1995,152 +1958,141 @@ void a64_hybrid_s8qa_dot_4x16_a55 (
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "uzp1 v20.8h, v20.8h, v21.8h\n"
-      "uzp1 v21.8h, v22.8h, v23.8h\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "uzp1 v24.8h, v24.8h, v25.8h\n"
-      "uzp1 v16.16b, v16.16b, v17.16b\n"
-      "uzp1 v20.16b, v20.16b, v21.16b\n"
       "smax v26.4s, v26.4s, v5.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
       "smax v31.4s, v31.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v20.8h, v20.8h, v21.8h\n"
+      "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
       "uzp1 v28.8h, v28.8h, v29.8h\n"
       "uzp1 v29.8h, v30.8h, v31.8h\n"
+      "cmp x15, #0x10\n"
+      "uzp1 v16.16b, v16.16b, v17.16b\n"
+      "uzp1 v20.16b, v20.16b, v21.16b\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 119f\n"
-      "tbz x11, #3, 114f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x11, #2, 112f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
-      "tbz x11, #1, 111f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "tbz x15, #3, 114f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x15, #2, 112f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
+      "tbz x15, #1, 111f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 118f\n"
       "111:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 118f\n"
       "112:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 113f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "tbz x15, #1, 113f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 118f\n"
       "113:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 118f\n"
       "114:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 116f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
-      "tbz x11, #1, 115f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "tbz x15, #2, 116f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
+      "tbz x15, #1, 115f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 118f\n"
       "115:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 118f\n"
       "116:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 117f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "tbz x15, #1, 117f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 118f\n"
       "117:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "118:"  // Height 4: Partial direct writeback: Done
       "b 120f\n"
       "119:"  // Height 4: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "120:"  // Height 4: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 92b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 122f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "121:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "122:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp
index a1c4b34d38..485a47dc67 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,69 +85,69 @@ void a64_hybrid_s8qa_dot_4x16 (
       "cmp %x[M], #0x2\n"
       "bgt 61f\n"
       "beq 31f\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 11f\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 9f\n"
-      "7:"  // Height 1: Multiply loop: Main loop head
-      ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
-      ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q8, [x28, #0x40]\n"
-      ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x50]\n"
       "ldr q10, [x28, #0x60]\n"
-      ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
+      "blt 9f\n"
+      "7:"  // Height 1: Multiply loop: Main loop head
+      ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x70]\n"
+      ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
-      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
-      ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
+      ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
+      ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
+      ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
-      ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
+      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
+      ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0xd0]\n"
-      ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
-      ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
+      ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
       "ldr q4, [x28, #0xe0]\n"
+      ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
+      ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
       "add x28, x28, #0x100\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
@@ -157,42 +157,42 @@ void a64_hybrid_s8qa_dot_4x16 (
       "tbnz %x[flags], #31, 8f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "ldr q0, [x23, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q0, [x24, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
-      "bge 7b\n"
-      "9:"  // Height 1: Multiply loop: Single iteration only
-      ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
-      ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x30]\n"
       "ldr q8, [x28, #0x40]\n"
-      ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x50]\n"
-      ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "bge 7b\n"
+      "9:"  // Height 1: Multiply loop: Single iteration only
+      ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x70]\n"
-      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
+      ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
+      ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
-      ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
-      ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
+      ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0xa0]\n"
+      ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
-      ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
+      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
-      ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
+      ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0xd0]\n"
+      ".inst 0x4fa0e093  // sdot v19.4s, v4.16b, v0.4b[1]\n"
       "ldr q4, [x28, #0xe0]\n"
+      ".inst 0x4f80e8b0  // sdot v16.4s, v5.16b, v0.4b[2]\n"
+      "ldr q5, [x28, #0xf0]\n"
+      ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f80e913  // sdot v19.4s, v8.16b, v0.4b[2]\n"
-      "ldr q5, [x28, #0xf0]\n"
+      "add x24, x24, #0x10\n"
       "add x28, x28, #0x100\n"
       ".inst 0x4fa0e930  // sdot v16.4s, v9.16b, v0.4b[3]\n"
       ".inst 0x4fa0e951  // sdot v17.4s, v10.16b, v0.4b[3]\n"
@@ -201,83 +201,83 @@ void a64_hybrid_s8qa_dot_4x16 (
       "tbnz %x[flags], #31, 10f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 18f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 18f\n"
+      "cmp x25, #0x4\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
+      ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x4f80e133  // sdot v19.4s, v9.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       "bge 12b\n"
-      "cbz x24, 18f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 15f\n"
-      "ldr h0, [x23], #0x2\n"
-      "tbz x24, #0, 16f\n"
-      "ld1 { v0.b }[2], [x23]\n"
+      "cbz x25, 18f\n"
+      "tbz x25, #1, 15f\n"
+      "ldr h0, [x24], #0x2\n"
+      "tbz x25, #0, 16f\n"
+      "ld1 { v0.b }[2], [x24]\n"
       "b 16f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 17f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "17:"  // Height 1: Multiply loop: unique 4: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x4f80e0d3  // sdot v19.4s, v6.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       "18:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "tbnz %x[flags], #31, 19f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
-      "ld1r { v1.4s }, [x22]\n"
       "neg v1.4s, v1.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "19:"  // Height 1: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+      "add x10, x10, #0x40\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
@@ -289,100 +289,100 @@ void a64_hybrid_s8qa_dot_4x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "20:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "cmp x9, #0x10\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 29f\n"
       "tbz x9, #3, 24f\n"
-      "str d16, [x26], #0x8\n"
+      "str d16, [x27], #0x8\n"
       "tbz x9, #2, 22f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
       "tbz x9, #1, 21f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[14], [x26]\n"
+      "st1 { v16.b }[14], [x27]\n"
       "b 28f\n"
       "21:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[12], [x26]\n"
+      "st1 { v16.b }[12], [x27]\n"
       "b 28f\n"
       "22:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 23f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[10], [x26]\n"
+      "st1 { v16.b }[10], [x27]\n"
       "b 28f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[8], [x26]\n"
+      "st1 { v16.b }[8], [x27]\n"
       "b 28f\n"
       "24:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 26f\n"
-      "str s16, [x26], #0x4\n"
+      "str s16, [x27], #0x4\n"
       "tbz x9, #1, 25f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[6], [x26]\n"
+      "st1 { v16.b }[6], [x27]\n"
       "b 28f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[4], [x26]\n"
+      "st1 { v16.b }[4], [x27]\n"
       "b 28f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 27f\n"
-      "str h16, [x26], #0x2\n"
+      "str h16, [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[2], [x26]\n"
+      "st1 { v16.b }[2], [x27]\n"
       "b 28f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
       "28:"  // Height 1: Partial direct writeback: Done
       "b 30f\n"
       "29:"  // Height 1: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
       "30:"  // Height 1: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 2b\n"
       "b 122f\n"
       "31:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v15.16b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "32:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -393,57 +393,57 @@ void a64_hybrid_s8qa_dot_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "33:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "34:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 35f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 36f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 36f\n"
       "35:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "36:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 41f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 39f\n"
       "37:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
-      "ldr q4, [x28, #0x70]\n"
+      "ldr q6, [x28, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
+      "ldr q7, [x28, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x90]\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
+      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
@@ -474,40 +474,40 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "38:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "bge 37b\n"
       "39:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
-      "ldr q4, [x28, #0x70]\n"
+      "ldr q6, [x28, #0x90]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f7  // sdot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
+      "ldr q7, [x28, #0xa0]\n"
       ".inst 0x4fa0e110  // sdot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x90]\n"
       ".inst 0x4fa1e114  // sdot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
+      ".inst 0x4fa0e131  // sdot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x4fa1e135  // sdot v21.4s, v9.16b, v1.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
       ".inst 0x4fa0e152  // sdot v18.4s, v10.16b, v0.4b[1]\n"
@@ -538,104 +538,104 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 6: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "41:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 48f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 48f\n"
+      "cmp x25, #0x4\n"
       "blt 44f\n"
       "42:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
       "tbnz %x[flags], #31, 43f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "43:"  // Height 2: Multiply loop: unique 7: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x4f81e116  // sdot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x4f80e133  // sdot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x4f81e137  // sdot v23.4s, v9.16b, v1.4b[0]\n"
       "bge 42b\n"
-      "cbz x24, 48f\n"
       "44:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 45f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "tbz x24, #0, 46f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
+      "cbz x25, 48f\n"
+      "tbz x25, #1, 45f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "tbz x25, #0, 46f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
       "b 46f\n"
       "45:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
       "46:"  // Height 2: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 47f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       "47:"  // Height 2: Multiply loop: unique 8: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
-      "add x28, x28, #0x40\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x4f81e0b6  // sdot v22.4s, v5.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f80e0d3  // sdot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0d7  // sdot v23.4s, v6.16b, v1.4b[0]\n"
       "48:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 34b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "tbnz %x[flags], #31, 49f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v2.4s, v2.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "49:"  // Height 2: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v16.4s, v16.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
+      "add x10, x10, #0x40\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "add v20.4s, v20.4s, v0.4s\n"
@@ -653,154 +653,154 @@ void a64_hybrid_s8qa_dot_4x16 (
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "tbz %x[flags], #5, 50f\n"
       "and v4.16b, v16.16b, v0.16b\n"
-      "and v5.16b, v17.16b, v0.16b\n"
-      "and v6.16b, v18.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "sqadd v17.4s, v17.4s, v5.4s\n"
-      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "and v5.16b, v17.16b, v0.16b\n"
+      "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
+      "and v10.16b, v22.16b, v0.16b\n"
+      "and v4.16b, v23.16b, v0.16b\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v17.4s, v17.4s, v5.4s\n"
+      "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
-      "and v10.16b, v22.16b, v0.16b\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "50:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
+      "add v22.4s, v22.4s, v4.4s\n"
+      "add v23.4s, v23.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v20.4s, v20.4s, v6.4s\n"
       "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
       "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "add v22.4s, v22.4s, v4.4s\n"
-      "add v23.4s, v23.4s, v4.4s\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "uzp1 v16.16b, v16.16b, v17.16b\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 59f\n"
       "tbz x9, #3, 54f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
       "tbz x9, #2, 52f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
       "tbz x9, #1, 51f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 58f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 58f\n"
       "52:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 53f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 58f\n"
       "53:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 58f\n"
       "54:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 56f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
       "tbz x9, #1, 55f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 58f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 58f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 57f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 58f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "58:"  // Height 2: Partial direct writeback: Done
       "b 60f\n"
       "59:"  // Height 2: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
       "60:"  // Height 2: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 32b\n"
       "b 122f\n"
       "61:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "movi v15.16b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "62:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -815,56 +815,56 @@ void a64_hybrid_s8qa_dot_4x16 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "63:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "64:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 65f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 66f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 66f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 66f\n"
       "65:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "66:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 71f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 69f\n"
       "67:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "ldr q5, [x28, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
@@ -891,8 +891,8 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
@@ -918,37 +918,37 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "68:"  // Height 3: Multiply loop: unique 9: skip row sum
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x28, #0x0]\n"
       "bge 67b\n"
       "69:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f81e0d6  // sdot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
       ".inst 0x4f82e0da  // sdot v26.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
       ".inst 0x4f80e0f3  // sdot v19.4s, v7.16b, v0.4b[0]\n"
@@ -975,8 +975,8 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f81e8b4  // sdot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x4f82e8b8  // sdot v24.4s, v5.16b, v2.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x4f80e8d1  // sdot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x4f81e8d5  // sdot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d9  // sdot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f80e8f2  // sdot v18.4s, v7.16b, v0.4b[2]\n"
@@ -1002,33 +1002,33 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "70:"  // Height 3: Multiply loop: unique 10: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "71:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 78f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 78f\n"
+      "cmp x25, #0x4\n"
       "blt 74f\n"
       "72:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
       "tbnz %x[flags], #31, 73f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "73:"  // Height 3: Multiply loop: unique 11: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f9  // sdot v25.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f80e112  // sdot v18.4s, v8.16b, v0.4b[0]\n"
@@ -1038,21 +1038,21 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f81e137  // sdot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x4f82e13b  // sdot v27.4s, v9.16b, v2.4b[0]\n"
       "bge 72b\n"
-      "cbz x24, 78f\n"
       "74:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 75f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "tbz x24, #0, 76f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
+      "cbz x25, 78f\n"
+      "tbz x25, #1, 75f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "tbz x25, #0, 76f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
       "b 76f\n"
       "75:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
       "76:"  // Height 3: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
@@ -1060,16 +1060,16 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 12: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
-      ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x4f82e099  // sdot v25.4s, v4.16b, v2.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x4f81e0b6  // sdot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x4f82e0ba  // sdot v26.4s, v5.16b, v2.4b[0]\n"
@@ -1077,49 +1077,49 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f81e0d7  // sdot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0db  // sdot v27.4s, v6.16b, v2.4b[0]\n"
       "78:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 64b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbnz %x[flags], #31, 79f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v3.4s, v3.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "mul v11.4s, v11.4s, v3.4s\n"
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "79:"  // Height 3: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v16.4s, v16.4s, v0.4s\n"
@@ -1151,98 +1151,98 @@ void a64_hybrid_s8qa_dot_4x16 (
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
+      "and v7.16b, v19.16b, v0.16b\n"
+      "and v8.16b, v20.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "and v7.16b, v19.16b, v0.16b\n"
-      "and v8.16b, v20.16b, v0.16b\n"
-      "and v9.16b, v21.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v22.4s, v22.4s, v10.4s\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "80:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v20.8h, v20.8h, v21.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
@@ -1251,103 +1251,103 @@ void a64_hybrid_s8qa_dot_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 89f\n"
       "tbz x9, #3, 84f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x9, #2, 82f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x9, #1, 81f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 88f\n"
       "81:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 88f\n"
       "82:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 83f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 88f\n"
       "83:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 88f\n"
       "84:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 86f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x9, #1, 85f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 88f\n"
       "85:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 88f\n"
       "86:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 87f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 88f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "88:"  // Height 3: Partial direct writeback: Done
       "b 90f\n"
       "89:"  // Height 3: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "90:"  // Height 3: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 62b\n"
       "b 122f\n"
       "91:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x19, #0x4\n"
       "movi v15.16b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "92:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1366,59 +1366,59 @@ void a64_hybrid_s8qa_dot_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "93:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "94:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 95f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 96f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 96f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 96f\n"
       "95:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "96:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 101f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 99f\n"
       "97:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f83e09c  // sdot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f83e0bd  // sdot v29.4s, v5.16b, v3.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
@@ -1491,38 +1491,38 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "98:"  // Height 4: Multiply loop: unique 13: skip row sum
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x28, #0x0]\n"
       "bge 97b\n"
       "99:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e090  // sdot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x4f81e094  // sdot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f82e098  // sdot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f83e09c  // sdot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f80e0b1  // sdot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4f81e0b5  // sdot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f82e0b9  // sdot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x4f83e0bd  // sdot v29.4s, v5.16b, v3.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
       ".inst 0x4f80e0d2  // sdot v18.4s, v6.16b, v0.4b[0]\n"
@@ -1595,19 +1595,19 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "100:"  // Height 4: Multiply loop: unique 14: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "101:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 108f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 108f\n"
+      "cmp x25, #0x4\n"
       "blt 104f\n"
       "102:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "tbnz %x[flags], #31, 103f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942c  // sdot v12.4s, v1.16b, v15.16b\n"
@@ -1615,16 +1615,16 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "103:"  // Height 4: Multiply loop: unique 15: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f80e0d0  // sdot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x4f81e0d4  // sdot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x4f82e0d8  // sdot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0dc  // sdot v28.4s, v6.16b, v3.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f80e0f1  // sdot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0f5  // sdot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f9  // sdot v25.4s, v7.16b, v2.4b[0]\n"
@@ -1638,24 +1638,24 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f82e13b  // sdot v27.4s, v9.16b, v2.4b[0]\n"
       ".inst 0x4f83e13f  // sdot v31.4s, v9.16b, v3.4b[0]\n"
       "bge 102b\n"
-      "cbz x24, 108f\n"
       "104:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 105f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "ldr h3, [x20], #0x2\n"
-      "tbz x24, #0, 106f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
-      "ld1 { v3.b }[2], [x20]\n"
+      "cbz x25, 108f\n"
+      "tbz x25, #1, 105f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "ldr h3, [x21], #0x2\n"
+      "tbz x25, #0, 106f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
+      "ld1 { v3.b }[2], [x21]\n"
       "b 106f\n"
       "105:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
-      "ldr b3, [x20, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
+      "ldr b3, [x21, #0x0]\n"
       "106:"  // Height 4: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 107f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
@@ -1664,16 +1664,16 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4e8f946e  // sdot v14.4s, v3.16b, v15.16b\n"
       "107:"  // Height 4: Multiply loop: unique 16: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x4f80e150  // sdot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x4f81e154  // sdot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
-      ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x4f82e158  // sdot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x4f83e15c  // sdot v28.4s, v10.16b, v3.4b[0]\n"
       ".inst 0x4f80e091  // sdot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x4f81e095  // sdot v21.4s, v4.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x4f82e099  // sdot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x4f83e09d  // sdot v29.4s, v4.16b, v3.4b[0]\n"
       ".inst 0x4f80e0b2  // sdot v18.4s, v5.16b, v0.4b[0]\n"
@@ -1685,27 +1685,27 @@ void a64_hybrid_s8qa_dot_4x16 (
       ".inst 0x4f82e0db  // sdot v27.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0df  // sdot v31.4s, v6.16b, v3.4b[0]\n"
       "108:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 94b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "tbnz %x[flags], #31, 109f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v4.4s, v4.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
@@ -1714,25 +1714,25 @@ void a64_hybrid_s8qa_dot_4x16 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "109:"  // Height 4: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v28.4s, v28.4s, v14.4s\n"
@@ -1775,126 +1775,126 @@ void a64_hybrid_s8qa_dot_4x16 (
       "tbz %x[flags], #5, 110f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
-      "and v6.16b, v18.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
-      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
       "and v10.16b, v22.16b, v0.16b\n"
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
+      "sqadd v20.4s, v20.4s, v8.4s\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sqadd v25.4s, v25.4s, v6.4s\n"
-      "sqadd v26.4s, v26.4s, v7.4s\n"
-      "sqadd v27.4s, v27.4s, v8.4s\n"
       "and v9.16b, v28.16b, v0.16b\n"
       "and v10.16b, v29.16b, v0.16b\n"
       "and v4.16b, v30.16b, v0.16b\n"
+      "and v5.16b, v31.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v25.4s, v25.4s, v6.4s\n"
+      "sqadd v26.4s, v26.4s, v7.4s\n"
+      "sqadd v27.4s, v27.4s, v8.4s\n"
       "sqadd v28.4s, v28.4s, v9.4s\n"
       "sqadd v29.4s, v29.4s, v10.4s\n"
       "sqadd v30.4s, v30.4s, v4.4s\n"
-      "and v5.16b, v31.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v5.4s\n"
       "110:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
+      "srshl v28.4s, v28.4s, v0.4s\n"
+      "srshl v29.4s, v29.4s, v0.4s\n"
+      "srshl v30.4s, v30.4s, v0.4s\n"
+      "srshl v31.4s, v31.4s, v0.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "srshl v28.4s, v28.4s, v0.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
-      "srshl v29.4s, v29.4s, v0.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
-      "srshl v30.4s, v30.4s, v0.4s\n"
-      "srshl v31.4s, v31.4s, v0.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
       "smax v31.4s, v31.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
@@ -1907,120 +1907,120 @@ void a64_hybrid_s8qa_dot_4x16 (
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 119f\n"
       "tbz x9, #3, 114f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
       "tbz x9, #2, 112f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
       "tbz x9, #1, 111f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 118f\n"
       "111:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 118f\n"
       "112:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 113f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 118f\n"
       "113:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 118f\n"
       "114:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 116f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
       "tbz x9, #1, 115f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 118f\n"
       "115:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 118f\n"
       "116:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 117f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 118f\n"
       "117:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "118:"  // Height 4: Partial direct writeback: Done
       "b 120f\n"
       "119:"  // Height 4: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "120:"  // Height 4: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 92b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 122f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "121:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "122:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp
index 4bc807cd8e..69d01a265e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,13 +85,13 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "cmp %x[M], #0x2\n"
       "bgt 65f\n"
       "beq 33f\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -102,27 +102,27 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 11f\n"
-      "ldr q1, [x23, #0x0]\n"
+      "ldr q1, [x24, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "cmp x25, #0x20\n"
       "ldr q6, [x28, #0x10]\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
@@ -132,10 +132,9 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "blt 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
@@ -152,9 +151,10 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "ldr q5, [x28, #0xe0]\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x4e87a434  // smmla v20.4s, v1.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e88a431  // smmla v17.4s, v1.16b, v8.16b\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x4e89a435  // smmla v21.4s, v1.16b, v9.16b\n"
       ".inst 0x4e8aa432  // smmla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x4e84a436  // smmla v22.4s, v1.16b, v4.16b\n"
@@ -164,25 +164,23 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "ldr q1, [x23, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q1, [x24, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q6, [x28, #0x10]\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
       "ldr q9, [x28, #0x40]\n"
       "ldr q10, [x28, #0x50]\n"
       "ldr q4, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "bge 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "sub x24, x24, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
@@ -199,9 +197,11 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "ldr q5, [x28, #0xe0]\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x4e87a434  // smmla v20.4s, v1.16b, v7.16b\n"
       ".inst 0x4e88a431  // smmla v17.4s, v1.16b, v8.16b\n"
+      "add x24, x24, #0x10\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x4e89a435  // smmla v21.4s, v1.16b, v9.16b\n"
       ".inst 0x4e8aa432  // smmla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x4e84a436  // smmla v22.4s, v1.16b, v4.16b\n"
@@ -211,120 +211,118 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 20f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 20f\n"
+      "cmp x25, #0x8\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x23], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
-      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
+      "sub x25, x25, #0x8\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x30]\n"
+      "cmp x25, #0x8\n"
+      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x4e86a416  // smmla v22.4s, v0.16b, v6.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e87a413  // smmla v19.4s, v0.16b, v7.16b\n"
       ".inst 0x4e88a417  // smmla v23.4s, v0.16b, v8.16b\n"
+      "add x28, x28, #0x80\n"
       "bge 12b\n"
-      "cbz x24, 20f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 16f\n"
-      "ldr s1, [x23], #0x4\n"
-      "tbz x24, #1, 15f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[6], [x23]\n"
+      "cbz x25, 20f\n"
+      "tbz x25, #2, 16f\n"
+      "ldr s1, [x24], #0x4\n"
+      "tbz x25, #1, 15f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[6], [x24]\n"
       "b 18f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[4], [x23]\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[4], [x24]\n"
       "b 18f\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 17f\n"
-      "ldr h1, [x23], #0x2\n"
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[2], [x23]\n"
+      "tbz x25, #1, 17f\n"
+      "ldr h1, [x24], #0x2\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[2], [x24]\n"
       "b 18f\n"
       "17:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
       "18:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 19f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "19:"  // Height 1: Multiply loop: unique 4: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x4e84a414  // smmla v20.4s, v0.16b, v4.16b\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x4e85a411  // smmla v17.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
       "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
-      ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x60]\n"
-      "ldr q10, [x28, #0x70]\n"
       ".inst 0x4e87a412  // smmla v18.4s, v0.16b, v7.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e88a416  // smmla v22.4s, v0.16b, v8.16b\n"
+      "ldr q9, [x28, #0x60]\n"
+      "ldr q10, [x28, #0x70]\n"
       ".inst 0x4e89a413  // smmla v19.4s, v0.16b, v9.16b\n"
       ".inst 0x4e8aa417  // smmla v23.4s, v0.16b, v10.16b\n"
+      "add x28, x28, #0x80\n"
       "20:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "uzp1 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
       "uzp1 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "mov v23.16b, v16.16b\n"
       "tbnz %x[flags], #31, 21f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v1.4s }, [x22]\n"
-      "dup v11.4s, v11.s[0]\n"
       "neg v1.4s, v1.4s\n"
+      "dup v11.4s, v11.s[0]\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "21:"  // Height 1: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v23.4s, v23.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v23.4s, v23.4s, v0.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+      "add x10, x10, #0x40\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
@@ -336,100 +334,100 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "22:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "cmp x9, #0x10\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v23.8h, v23.8h, v17.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v23.8h, v23.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v23.16b, v23.16b, v17.16b\n"
       "bge 31f\n"
       "tbz x9, #3, 26f\n"
-      "str d23, [x26], #0x8\n"
+      "str d23, [x27], #0x8\n"
       "tbz x9, #2, 24f\n"
-      "st1 { v23.s }[2], [x26], #0x4\n"
+      "st1 { v23.s }[2], [x27], #0x4\n"
       "tbz x9, #1, 23f\n"
-      "st1 { v23.h }[6], [x26], #0x2\n"
+      "st1 { v23.h }[6], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[14], [x26]\n"
+      "st1 { v23.b }[14], [x27]\n"
       "b 30f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[12], [x26]\n"
+      "st1 { v23.b }[12], [x27]\n"
       "b 30f\n"
       "24:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 25f\n"
-      "st1 { v23.h }[4], [x26], #0x2\n"
+      "st1 { v23.h }[4], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[10], [x26]\n"
+      "st1 { v23.b }[10], [x27]\n"
       "b 30f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[8], [x26]\n"
+      "st1 { v23.b }[8], [x27]\n"
       "b 30f\n"
       "26:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 28f\n"
-      "str s23, [x26], #0x4\n"
+      "str s23, [x27], #0x4\n"
       "tbz x9, #1, 27f\n"
-      "st1 { v23.h }[2], [x26], #0x2\n"
+      "st1 { v23.h }[2], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[6], [x26]\n"
+      "st1 { v23.b }[6], [x27]\n"
       "b 30f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[4], [x26]\n"
+      "st1 { v23.b }[4], [x27]\n"
       "b 30f\n"
       "28:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 29f\n"
-      "str h23, [x26], #0x2\n"
+      "str h23, [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[2], [x26]\n"
+      "st1 { v23.b }[2], [x27]\n"
       "b 30f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b23, [x26, #0x0]\n"
+      "str b23, [x27, #0x0]\n"
       "30:"  // Height 1: Partial direct writeback: Done
       "b 32f\n"
       "31:"  // Height 1: Full writeback
-      "str q23, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
+      "str q23, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
       "32:"  // Height 1: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 2b\n"
       "b 130f\n"
       "33:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v15.16b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "34:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -440,64 +438,64 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "35:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "36:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 37f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 38f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 38f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 38f\n"
       "37:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "38:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 43f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 41f\n"
-      "39:"  // Height 2: Multiply loop: Main loop head
-      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
-      ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q9, [x28, #0x40]\n"
-      ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       "ldr q10, [x28, #0x50]\n"
-      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       "ldr q4, [x28, #0x60]\n"
-      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
+      "blt 41f\n"
+      "39:"  // Height 2: Multiply loop: Main loop head
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
-      ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
-      ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
-      ".inst 0x4e85a417  // smmla v23.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
+      ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
+      ".inst 0x4e85a417  // smmla v23.4s, v0.16b, v5.16b\n"
+      "ldr q5, [x28, #0xe0]\n"
+      ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
+      "ldr q6, [x28, #0xf0]\n"
       ".inst 0x4e87a434  // smmla v20.4s, v1.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e88a431  // smmla v17.4s, v1.16b, v8.16b\n"
-      "ldr q5, [x28, #0xe0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e89a435  // smmla v21.4s, v1.16b, v9.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
       "add x28, x28, #0x100\n"
       ".inst 0x4e8aa432  // smmla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x4e84a436  // smmla v22.4s, v1.16b, v4.16b\n"
@@ -507,49 +505,49 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "bge 39b\n"
-      "41:"  // Height 2: Multiply loop: Single iteration only
-      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x0]\n"
-      "sub x24, x24, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q7, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q8, [x28, #0x30]\n"
       "ldr q9, [x28, #0x40]\n"
-      ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       "ldr q10, [x28, #0x50]\n"
       "ldr q4, [x28, #0x60]\n"
-      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
-      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "bge 39b\n"
+      "41:"  // Height 2: Multiply loop: Single iteration only
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
-      ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       "ldr q7, [x28, #0x90]\n"
-      ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
-      ".inst 0x4e85a417  // smmla v23.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
+      ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
-      ".inst 0x4e87a434  // smmla v20.4s, v1.16b, v7.16b\n"
+      ".inst 0x4e85a417  // smmla v23.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0xe0]\n"
+      ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
+      "ldr q6, [x28, #0xf0]\n"
+      "sub x25, x25, #0x10\n"
+      ".inst 0x4e87a434  // smmla v20.4s, v1.16b, v7.16b\n"
       ".inst 0x4e88a431  // smmla v17.4s, v1.16b, v8.16b\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e89a435  // smmla v21.4s, v1.16b, v9.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x4e8aa432  // smmla v18.4s, v1.16b, v10.16b\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x4e84a436  // smmla v22.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
       ".inst 0x4e86a437  // smmla v23.4s, v1.16b, v6.16b\n"
@@ -557,136 +555,136 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       "42:"  // Height 2: Multiply loop: unique 6: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "43:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 52f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 52f\n"
+      "cmp x25, #0x8\n"
       "blt 46f\n"
       "44:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 45f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "45:"  // Height 2: Multiply loop: unique 7: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
-      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
+      "sub x25, x25, #0x8\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x30]\n"
+      "cmp x25, #0x8\n"
+      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x4e86a416  // smmla v22.4s, v0.16b, v6.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e87a413  // smmla v19.4s, v0.16b, v7.16b\n"
       ".inst 0x4e88a417  // smmla v23.4s, v0.16b, v8.16b\n"
+      "add x28, x28, #0x80\n"
       "bge 44b\n"
-      "cbz x24, 52f\n"
       "46:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 48f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "tbz x24, #1, 47f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
+      "cbz x25, 52f\n"
+      "tbz x25, #2, 48f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "tbz x25, #1, 47f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
       "b 50f\n"
       "47:"  // Height 2: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
       "b 50f\n"
       "48:"  // Height 2: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 49f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
+      "tbz x25, #1, 49f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
       "b 50f\n"
       "49:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
       "50:"  // Height 2: Multiply loop: Ragged operand read: Done
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 51f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       "51:"  // Height 2: Multiply loop: unique 8: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x4e84a414  // smmla v20.4s, v0.16b, v4.16b\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x4e85a411  // smmla v17.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
       "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
-      ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x60]\n"
-      "ldr q10, [x28, #0x70]\n"
       ".inst 0x4e87a412  // smmla v18.4s, v0.16b, v7.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e88a416  // smmla v22.4s, v0.16b, v8.16b\n"
+      "ldr q9, [x28, #0x60]\n"
+      "ldr q10, [x28, #0x70]\n"
       ".inst 0x4e89a413  // smmla v19.4s, v0.16b, v9.16b\n"
       ".inst 0x4e8aa417  // smmla v23.4s, v0.16b, v10.16b\n"
+      "add x28, x28, #0x80\n"
       "52:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 36b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "mov v23.16b, v4.16b\n"
       "tbnz %x[flags], #31, 53f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
+      "neg v2.4s, v2.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v2.4s, v2.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "53:"  // Height 2: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v23.4s, v23.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v23.4s, v23.4s, v0.4s\n"
       "add v20.4s, v20.4s, v1.4s\n"
+      "add x10, x10, #0x40\n"
       "add v21.4s, v21.4s, v2.4s\n"
       "add v22.4s, v22.4s, v3.4s\n"
       "add v16.4s, v16.4s, v0.4s\n"
@@ -704,154 +702,154 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "tbz %x[flags], #5, 54f\n"
       "and v4.16b, v23.16b, v0.16b\n"
-      "and v5.16b, v20.16b, v0.16b\n"
-      "and v6.16b, v21.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
-      "sqadd v20.4s, v20.4s, v5.4s\n"
-      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "and v5.16b, v20.16b, v0.16b\n"
+      "and v6.16b, v21.16b, v0.16b\n"
       "and v7.16b, v22.16b, v0.16b\n"
       "and v8.16b, v16.16b, v0.16b\n"
       "and v9.16b, v17.16b, v0.16b\n"
+      "and v10.16b, v18.16b, v0.16b\n"
+      "and v4.16b, v19.16b, v0.16b\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v20.4s, v20.4s, v5.4s\n"
+      "sqadd v21.4s, v21.4s, v6.4s\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "sqadd v16.4s, v16.4s, v8.4s\n"
       "sqadd v17.4s, v17.4s, v9.4s\n"
-      "and v10.16b, v18.16b, v0.16b\n"
-      "and v4.16b, v19.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v18.4s, v18.4s, v10.4s\n"
       "sqadd v19.4s, v19.4s, v4.4s\n"
       "54:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v23.8h, v23.8h, v20.8h\n"
       "uzp1 v20.8h, v21.8h, v22.8h\n"
-      "add v18.4s, v18.4s, v4.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v23.16b, v23.16b, v20.16b\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v23.16b, v23.16b, v20.16b\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 63f\n"
       "tbz x9, #3, 58f\n"
-      "str d23, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
+      "str d23, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
       "tbz x9, #2, 56f\n"
-      "st1 { v23.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
+      "st1 { v23.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
       "tbz x9, #1, 55f\n"
-      "st1 { v23.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
+      "st1 { v23.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
+      "st1 { v23.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
       "b 62f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
+      "st1 { v23.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
       "b 62f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 57f\n"
-      "st1 { v23.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
+      "st1 { v23.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
+      "st1 { v23.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
       "b 62f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
+      "st1 { v23.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
       "b 62f\n"
       "58:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 60f\n"
-      "str s23, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
+      "str s23, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
       "tbz x9, #1, 59f\n"
-      "st1 { v23.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
+      "st1 { v23.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
+      "st1 { v23.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
       "b 62f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
+      "st1 { v23.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
       "b 62f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 61f\n"
-      "str h23, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
+      "str h23, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
+      "st1 { v23.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
       "b 62f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b23, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
+      "str b23, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
       "62:"  // Height 2: Partial direct writeback: Done
       "b 64f\n"
       "63:"  // Height 2: Full writeback
-      "str q23, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
+      "str q23, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
       "64:"  // Height 2: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 34b\n"
       "b 130f\n"
       "65:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "movi v15.16b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "66:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -870,65 +868,65 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "67:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "68:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 69f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 70f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 70f\n"
       "69:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "70:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 75f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 73f\n"
-      "71:"  // Height 3: Multiply loop: Main loop head
-      "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x22, x22, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q7, [x28, #0x20]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
       "ldr q8, [x28, #0x30]\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
       "ldr q9, [x28, #0x40]\n"
       "ldr q10, [x28, #0x50]\n"
+      "blt 73f\n"
+      "71:"  // Height 3: Multiply loop: Main loop head
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
+      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q4, [x28, #0x60]\n"
+      ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
-      "ldr q6, [x28, #0x80]\n"
-      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45d  // smmla v29.4s, v2.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x4e89a45a  // smmla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45e  // smmla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
@@ -962,49 +960,49 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f946d  // sdot v13.4s, v3.16b, v15.16b\n"
       "72:"  // Height 3: Multiply loop: unique 9: skip row sum
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
       "bge 71b\n"
       "73:"  // Height 3: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "sub x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      "ldr q9, [x28, #0x40]\n"
-      "ldr q10, [x28, #0x50]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q4, [x28, #0x60]\n"
+      ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
-      "ldr q6, [x28, #0x80]\n"
-      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      "sub x25, x25, #0x10\n"
+      ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45d  // smmla v29.4s, v2.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x4e89a45a  // smmla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45e  // smmla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
       ".inst 0x4e84a45b  // smmla v27.4s, v2.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
@@ -1035,43 +1033,42 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f946d  // sdot v13.4s, v3.16b, v15.16b\n"
       "74:"  // Height 3: Multiply loop: unique 10: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "75:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 84f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 84f\n"
+      "cmp x25, #0x8\n"
       "blt 78f\n"
       "76:"  // Height 3: Multiply loop: Odd block loop
-      "movi v7.16b, #0x0\n"
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x21], #0x8\n"
+      "ldr d3, [x22], #0x8\n"
       "trn1 v2.2d, v3.2d, v7.2d\n"
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 11: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
+      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a458  // smmla v24.4s, v2.16b, v8.16b\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
       "ldr q4, [x28, #0x30]\n"
-      ".inst 0x4e89a45c  // smmla v28.4s, v2.16b, v9.16b\n"
+      "sub x25, x25, #0x8\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x4e8aa459  // smmla v25.4s, v2.16b, v10.16b\n"
+      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x4e89a45c  // smmla v28.4s, v2.16b, v9.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x4e8aa459  // smmla v25.4s, v2.16b, v10.16b\n"
       ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e84a45d  // smmla v29.4s, v2.16b, v4.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x4e85a45a  // smmla v26.4s, v2.16b, v5.16b\n"
       ".inst 0x4e86a416  // smmla v22.4s, v0.16b, v6.16b\n"
@@ -1081,43 +1078,42 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e88a417  // smmla v23.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45f  // smmla v31.4s, v2.16b, v8.16b\n"
       "bge 76b\n"
-      "cbz x24, 84f\n"
       "78:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 80f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "ldr s3, [x21], #0x4\n"
-      "tbz x24, #1, 79f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "ld1 { v3.h }[2], [x21], #0x2\n"
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
-      "ld1 { v3.b }[6], [x21]\n"
+      "cbz x25, 84f\n"
+      "tbz x25, #2, 80f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "ldr s3, [x22], #0x4\n"
+      "tbz x25, #1, 79f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "ld1 { v3.h }[2], [x22], #0x2\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
+      "ld1 { v3.b }[6], [x22]\n"
       "b 82f\n"
       "79:"  // Height 3: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
-      "ld1 { v3.b }[4], [x21]\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
+      "ld1 { v3.b }[4], [x22]\n"
       "b 82f\n"
       "80:"  // Height 3: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 81f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "ldr h3, [x21], #0x2\n"
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
-      "ld1 { v3.b }[2], [x21]\n"
+      "tbz x25, #1, 81f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "ldr h3, [x22], #0x2\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
+      "ld1 { v3.b }[2], [x22]\n"
       "b 82f\n"
       "81:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
-      "ldr b3, [x21, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
+      "ldr b3, [x22, #0x0]\n"
       "82:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v9.16b, #0x0\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v9.2d\n"
       "tbnz %x[flags], #31, 83f\n"
@@ -1125,24 +1121,24 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "83:"  // Height 3: Multiply loop: unique 12: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa458  // smmla v24.4s, v2.16b, v10.16b\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x4e84a414  // smmla v20.4s, v0.16b, v4.16b\n"
-      "ldr q7, [x28, #0x40]\n"
       ".inst 0x4e84a45c  // smmla v28.4s, v2.16b, v4.16b\n"
+      "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
       ".inst 0x4e85a411  // smmla v17.4s, v0.16b, v5.16b\n"
-      "ldr q9, [x28, #0x60]\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
+      "ldr q9, [x28, #0x60]\n"
       "ldr q10, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a45d  // smmla v29.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a412  // smmla v18.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a45a  // smmla v26.4s, v2.16b, v7.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x4e88a416  // smmla v22.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45e  // smmla v30.4s, v2.16b, v8.16b\n"
       ".inst 0x4e89a413  // smmla v19.4s, v0.16b, v9.16b\n"
@@ -1150,21 +1146,21 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8aa417  // smmla v23.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45f  // smmla v31.4s, v2.16b, v10.16b\n"
       "84:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 68b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
-      "add x20, x21, x19\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
@@ -1174,37 +1170,37 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "mov v31.16b, v4.16b\n"
       "tbnz %x[flags], #31, 85f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
       "addp v13.4s, v13.4s, v13.4s\n"
+      "neg v3.4s, v3.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v3.4s, v3.4s\n"
       "dup v13.4s, v13.s[0]\n"
       "mul v11.4s, v11.4s, v3.4s\n"
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "85:"  // Height 3: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v31.4s, v31.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v31.4s, v31.4s, v0.4s\n"
@@ -1236,98 +1232,98 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v20.16b, v0.16b\n"
       "and v6.16b, v21.16b, v0.16b\n"
+      "and v7.16b, v22.16b, v0.16b\n"
+      "and v8.16b, v16.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
       "sqadd v21.4s, v21.4s, v6.4s\n"
-      "and v7.16b, v22.16b, v0.16b\n"
-      "and v8.16b, v16.16b, v0.16b\n"
-      "and v9.16b, v17.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "sqadd v16.4s, v16.4s, v8.4s\n"
-      "sqadd v17.4s, v17.4s, v9.4s\n"
+      "and v9.16b, v17.16b, v0.16b\n"
       "and v10.16b, v18.16b, v0.16b\n"
       "and v4.16b, v19.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v18.4s, v18.4s, v10.4s\n"
-      "sqadd v19.4s, v19.4s, v4.4s\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v17.4s, v17.4s, v9.4s\n"
+      "sqadd v18.4s, v18.4s, v10.4s\n"
+      "sqadd v19.4s, v19.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "86:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "uzp1 v31.8h, v31.8h, v20.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "uzp1 v20.8h, v21.8h, v22.8h\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v31.8h, v31.8h, v20.8h\n"
+      "uzp1 v20.8h, v21.8h, v22.8h\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
@@ -1336,103 +1332,103 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 95f\n"
       "tbz x9, #3, 90f\n"
-      "str d31, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d31, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x9, #2, 88f\n"
-      "st1 { v31.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v31.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x9, #1, 87f\n"
-      "st1 { v31.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v31.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v31.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 94f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v31.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 94f\n"
       "88:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 89f\n"
-      "st1 { v31.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v31.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v31.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 94f\n"
       "89:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v31.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 94f\n"
       "90:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 92f\n"
-      "str s31, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s31, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x9, #1, 91f\n"
-      "st1 { v31.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v31.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v31.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 94f\n"
       "91:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v31.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 94f\n"
       "92:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 93f\n"
-      "str h31, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h31, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v31.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 94f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b31, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b31, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "94:"  // Height 3: Partial direct writeback: Done
       "b 96f\n"
       "95:"  // Height 3: Full writeback
-      "str q31, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q31, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "96:"  // Height 3: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 66b\n"
       "b 130f\n"
       "97:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x19, #0x4\n"
       "movi v15.16b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "98:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1451,70 +1447,70 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "99:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "100:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 101f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 102f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 102f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 102f\n"
       "101:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "102:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 107f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
       "blt 105f\n"
       "103:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
+      "add x24, x24, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q4, [x28, #0x60]\n"
       ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x40]\n"
       ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
-      "ldr q10, [x28, #0x50]\n"
-      "ldr q4, [x28, #0x60]\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
+      "ldr q7, [x28, #0x90]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
-      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e88a45d  // smmla v29.4s, v2.16b, v8.16b\n"
-      "ldr q7, [x28, #0x90]\n"
-      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x22, x22, #0x10\n"
+      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x4e89a45a  // smmla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45e  // smmla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
@@ -1548,52 +1544,52 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f946d  // sdot v13.4s, v3.16b, v15.16b\n"
       "104:"  // Height 4: Multiply loop: unique 13: skip row sum
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
       "bge 103b\n"
       "105:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "sub x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
+      "sub x25, x25, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x4e85a410  // smmla v16.4s, v0.16b, v5.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q4, [x28, #0x60]\n"
       ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x4e86a414  // smmla v20.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x40]\n"
       ".inst 0x4e86a45c  // smmla v28.4s, v2.16b, v6.16b\n"
-      "ldr q10, [x28, #0x50]\n"
-      "ldr q4, [x28, #0x60]\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e87a411  // smmla v17.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
+      "ldr q7, [x28, #0x90]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e88a415  // smmla v21.4s, v0.16b, v8.16b\n"
-      "ldr q6, [x28, #0x80]\n"
       ".inst 0x4e88a45d  // smmla v29.4s, v2.16b, v8.16b\n"
-      "ldr q7, [x28, #0x90]\n"
-      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0x4e89a412  // smmla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x4e89a45a  // smmla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e8aa416  // smmla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45e  // smmla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4e84a413  // smmla v19.4s, v0.16b, v4.16b\n"
       ".inst 0x4e84a45b  // smmla v27.4s, v2.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
@@ -1624,44 +1620,44 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f942b  // sdot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x4e8f946d  // sdot v13.4s, v3.16b, v15.16b\n"
       "106:"  // Height 4: Multiply loop: unique 14: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "107:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 116f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 116f\n"
+      "cmp x25, #0x8\n"
       "blt 110f\n"
       "108:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "ldr d3, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v2.2d, v3.2d, v7.2d\n"
       "tbnz %x[flags], #31, 109f\n"
       ".inst 0x4e8f940b  // sdot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "109:"  // Height 4: Multiply loop: unique 15: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
+      ".inst 0x4e88a410  // smmla v16.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a458  // smmla v24.4s, v2.16b, v8.16b\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
       "ldr q4, [x28, #0x30]\n"
-      ".inst 0x4e89a45c  // smmla v28.4s, v2.16b, v9.16b\n"
+      "sub x25, x25, #0x8\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x4e8aa459  // smmla v25.4s, v2.16b, v10.16b\n"
+      ".inst 0x4e89a414  // smmla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x4e89a45c  // smmla v28.4s, v2.16b, v9.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x4e8aa411  // smmla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x4e8aa459  // smmla v25.4s, v2.16b, v10.16b\n"
       ".inst 0x4e84a415  // smmla v21.4s, v0.16b, v4.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e84a45d  // smmla v29.4s, v2.16b, v4.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x4e85a412  // smmla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x4e85a45a  // smmla v26.4s, v2.16b, v5.16b\n"
       ".inst 0x4e86a416  // smmla v22.4s, v0.16b, v6.16b\n"
@@ -1671,48 +1667,48 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e88a417  // smmla v23.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45f  // smmla v31.4s, v2.16b, v8.16b\n"
       "bge 108b\n"
-      "cbz x24, 116f\n"
       "110:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 112f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "ldr s3, [x21], #0x4\n"
-      "ldr s9, [x20], #0x4\n"
-      "tbz x24, #1, 111f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "ld1 { v3.h }[2], [x21], #0x2\n"
-      "ld1 { v9.h }[2], [x20], #0x2\n"
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
-      "ld1 { v3.b }[6], [x21]\n"
-      "ld1 { v9.b }[6], [x20]\n"
+      "cbz x25, 116f\n"
+      "tbz x25, #2, 112f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "ldr s3, [x22], #0x4\n"
+      "ldr s9, [x21], #0x4\n"
+      "tbz x25, #1, 111f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "ld1 { v3.h }[2], [x22], #0x2\n"
+      "ld1 { v9.h }[2], [x21], #0x2\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
+      "ld1 { v3.b }[6], [x22]\n"
+      "ld1 { v9.b }[6], [x21]\n"
       "b 114f\n"
       "111:"  // Height 4: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
-      "ld1 { v3.b }[4], [x21]\n"
-      "ld1 { v9.b }[4], [x20]\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
+      "ld1 { v3.b }[4], [x22]\n"
+      "ld1 { v9.b }[4], [x21]\n"
       "b 114f\n"
       "112:"  // Height 4: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 113f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "ldr h3, [x21], #0x2\n"
-      "ldr h9, [x20], #0x2\n"
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
-      "ld1 { v3.b }[2], [x21]\n"
-      "ld1 { v9.b }[2], [x20]\n"
+      "tbz x25, #1, 113f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "ldr h3, [x22], #0x2\n"
+      "ldr h9, [x21], #0x2\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
+      "ld1 { v3.b }[2], [x22]\n"
+      "ld1 { v9.b }[2], [x21]\n"
       "b 114f\n"
       "113:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
-      "ldr b3, [x21, #0x0]\n"
-      "ldr b9, [x20, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
+      "ldr b3, [x22, #0x0]\n"
+      "ldr b9, [x21, #0x0]\n"
       "114:"  // Height 4: Multiply loop: Ragged operand read: Done
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v9.2d\n"
@@ -1721,24 +1717,24 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8f944d  // sdot v13.4s, v2.16b, v15.16b\n"
       "115:"  // Height 4: Multiply loop: unique 16: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x4e8aa410  // smmla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa458  // smmla v24.4s, v2.16b, v10.16b\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x4e84a414  // smmla v20.4s, v0.16b, v4.16b\n"
-      "ldr q7, [x28, #0x40]\n"
       ".inst 0x4e84a45c  // smmla v28.4s, v2.16b, v4.16b\n"
+      "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
       ".inst 0x4e85a411  // smmla v17.4s, v0.16b, v5.16b\n"
-      "ldr q9, [x28, #0x60]\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
+      "ldr q9, [x28, #0x60]\n"
       "ldr q10, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x4e86a415  // smmla v21.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a45d  // smmla v29.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a412  // smmla v18.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a45a  // smmla v26.4s, v2.16b, v7.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x4e88a416  // smmla v22.4s, v0.16b, v8.16b\n"
       ".inst 0x4e88a45e  // smmla v30.4s, v2.16b, v8.16b\n"
       ".inst 0x4e89a413  // smmla v19.4s, v0.16b, v9.16b\n"
@@ -1746,25 +1742,25 @@ void a64_hybrid_s8qa_mmla_4x16 (
       ".inst 0x4e8aa417  // smmla v23.4s, v0.16b, v10.16b\n"
       ".inst 0x4e8aa45f  // smmla v31.4s, v2.16b, v10.16b\n"
       "116:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 100b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
-      "add x20, x21, x19\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
+      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "uzp1 v23.2d, v24.2d, v28.2d\n"
       "uzp2 v24.2d, v24.2d, v28.2d\n"
@@ -1776,13 +1772,13 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "mov v31.16b, v4.16b\n"
       "tbnz %x[flags], #31, 117f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "addp v13.4s, v13.4s, v13.4s\n"
+      "neg v4.4s, v4.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v4.4s, v4.4s\n"
       "dup v14.4s, v13.s[3]\n"
       "dup v13.4s, v13.s[0]\n"
       "mul v11.4s, v11.4s, v4.4s\n"
@@ -1790,25 +1786,25 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "117:"  // Height 4: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v31.4s, v31.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v23.4s, v23.4s, v13.4s\n"
       "add v28.4s, v28.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v29.4s, v29.4s, v13.4s\n"
       "add v30.4s, v30.4s, v13.4s\n"
       "add v24.4s, v24.4s, v14.4s\n"
@@ -1851,126 +1847,126 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "tbz %x[flags], #5, 118f\n"
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v20.16b, v0.16b\n"
-      "and v6.16b, v21.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
-      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "and v6.16b, v21.16b, v0.16b\n"
       "and v7.16b, v22.16b, v0.16b\n"
       "and v8.16b, v16.16b, v0.16b\n"
       "and v9.16b, v17.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sqadd v22.4s, v22.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v8.4s\n"
-      "sqadd v17.4s, v17.4s, v9.4s\n"
       "and v10.16b, v18.16b, v0.16b\n"
       "and v4.16b, v19.16b, v0.16b\n"
       "and v5.16b, v23.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "sqadd v22.4s, v22.4s, v7.4s\n"
+      "sqadd v16.4s, v16.4s, v8.4s\n"
+      "sqadd v17.4s, v17.4s, v9.4s\n"
       "sqadd v18.4s, v18.4s, v10.4s\n"
       "sqadd v19.4s, v19.4s, v4.4s\n"
       "sqadd v23.4s, v23.4s, v5.4s\n"
       "and v6.16b, v28.16b, v0.16b\n"
       "and v7.16b, v29.16b, v0.16b\n"
       "and v8.16b, v30.16b, v0.16b\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sqadd v28.4s, v28.4s, v6.4s\n"
-      "sqadd v29.4s, v29.4s, v7.4s\n"
-      "sqadd v30.4s, v30.4s, v8.4s\n"
       "and v9.16b, v24.16b, v0.16b\n"
       "and v10.16b, v25.16b, v0.16b\n"
       "and v4.16b, v26.16b, v0.16b\n"
+      "and v5.16b, v27.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v28.4s, v28.4s, v6.4s\n"
+      "sqadd v29.4s, v29.4s, v7.4s\n"
+      "sqadd v30.4s, v30.4s, v8.4s\n"
       "sqadd v24.4s, v24.4s, v9.4s\n"
       "sqadd v25.4s, v25.4s, v10.4s\n"
       "sqadd v26.4s, v26.4s, v4.4s\n"
-      "and v5.16b, v27.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v27.4s, v27.4s, v5.4s\n"
       "118:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "srshl v28.4s, v28.4s, v0.4s\n"
+      "srshl v29.4s, v29.4s, v0.4s\n"
+      "srshl v30.4s, v30.4s, v0.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v28.4s, v28.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "srshl v29.4s, v29.4s, v0.4s\n"
-      "srshl v30.4s, v30.4s, v0.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "add v24.4s, v24.4s, v4.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
-      "smin v24.4s, v24.4s, v6.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "uzp1 v31.8h, v31.8h, v20.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v20.8h, v21.8h, v22.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v31.8h, v31.8h, v20.8h\n"
+      "uzp1 v20.8h, v21.8h, v22.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v23.8h, v23.8h, v28.8h\n"
@@ -1983,120 +1979,120 @@ void a64_hybrid_s8qa_mmla_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 127f\n"
       "tbz x9, #3, 122f\n"
-      "str d31, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "str d24, [x19], #0x8\n"
+      "str d31, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "str d24, [x20], #0x8\n"
       "tbz x9, #2, 120f\n"
-      "st1 { v31.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v23.s }[2], [x20], #0x4\n"
-      "st1 { v24.s }[2], [x19], #0x4\n"
+      "st1 { v31.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v23.s }[2], [x21], #0x4\n"
+      "st1 { v24.s }[2], [x20], #0x4\n"
       "tbz x9, #1, 119f\n"
-      "st1 { v31.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v23.h }[6], [x20], #0x2\n"
-      "st1 { v24.h }[6], [x19], #0x2\n"
+      "st1 { v31.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v23.h }[6], [x21], #0x2\n"
+      "st1 { v24.h }[6], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v23.b }[14], [x20]\n"
-      "st1 { v24.b }[14], [x19]\n"
+      "st1 { v31.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v23.b }[14], [x21]\n"
+      "st1 { v24.b }[14], [x20]\n"
       "b 126f\n"
       "119:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v23.b }[12], [x20]\n"
-      "st1 { v24.b }[12], [x19]\n"
+      "st1 { v31.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v23.b }[12], [x21]\n"
+      "st1 { v24.b }[12], [x20]\n"
       "b 126f\n"
       "120:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 121f\n"
-      "st1 { v31.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v23.h }[4], [x20], #0x2\n"
-      "st1 { v24.h }[4], [x19], #0x2\n"
+      "st1 { v31.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v23.h }[4], [x21], #0x2\n"
+      "st1 { v24.h }[4], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v23.b }[10], [x20]\n"
-      "st1 { v24.b }[10], [x19]\n"
+      "st1 { v31.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v23.b }[10], [x21]\n"
+      "st1 { v24.b }[10], [x20]\n"
       "b 126f\n"
       "121:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v23.b }[8], [x20]\n"
-      "st1 { v24.b }[8], [x19]\n"
+      "st1 { v31.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v23.b }[8], [x21]\n"
+      "st1 { v24.b }[8], [x20]\n"
       "b 126f\n"
       "122:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 124f\n"
-      "str s31, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s23, [x20], #0x4\n"
-      "str s24, [x19], #0x4\n"
+      "str s31, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s23, [x21], #0x4\n"
+      "str s24, [x20], #0x4\n"
       "tbz x9, #1, 123f\n"
-      "st1 { v31.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v23.h }[2], [x20], #0x2\n"
-      "st1 { v24.h }[2], [x19], #0x2\n"
+      "st1 { v31.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v23.h }[2], [x21], #0x2\n"
+      "st1 { v24.h }[2], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v23.b }[6], [x20]\n"
-      "st1 { v24.b }[6], [x19]\n"
+      "st1 { v31.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v23.b }[6], [x21]\n"
+      "st1 { v24.b }[6], [x20]\n"
       "b 126f\n"
       "123:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v23.b }[4], [x20]\n"
-      "st1 { v24.b }[4], [x19]\n"
+      "st1 { v31.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v23.b }[4], [x21]\n"
+      "st1 { v24.b }[4], [x20]\n"
       "b 126f\n"
       "124:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 125f\n"
-      "str h31, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h23, [x20], #0x2\n"
-      "str h24, [x19], #0x2\n"
+      "str h31, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h23, [x21], #0x2\n"
+      "str h24, [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v23.b }[2], [x20]\n"
-      "st1 { v24.b }[2], [x19]\n"
+      "st1 { v31.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v23.b }[2], [x21]\n"
+      "st1 { v24.b }[2], [x20]\n"
       "b 126f\n"
       "125:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b31, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b23, [x20, #0x0]\n"
-      "str b24, [x19, #0x0]\n"
+      "str b31, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b23, [x21, #0x0]\n"
+      "str b24, [x20, #0x0]\n"
       "126:"  // Height 4: Partial direct writeback: Done
       "b 128f\n"
       "127:"  // Height 4: Full writeback
-      "str q31, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
-      "str q23, [x20, #0x0]\n"
-      "str q24, [x19, #0x0]\n"
+      "str q31, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
+      "str q23, [x21, #0x0]\n"
+      "str q24, [x20, #0x0]\n"
       "128:"  // Height 4: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 98b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 130f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 129f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "129:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "130:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp
index ba8a2ccb1d..b97b63cdce 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -95,11 +95,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 55f\n"
       "beq 28f\n"
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x16, %x[output_ptr]\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
       "movi v8.4s, #0x0\n"
@@ -110,15 +110,15 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
       "cbnz x14, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
       "mov x12, %x[input_ptr]\n"
@@ -126,93 +126,90 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "cmp x13, #0x10\n"
       "blt 9f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q6, [x15, #0x0]\n"
       "cmp x13, #0x20\n"
+      "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 8f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
-      "ldr x11, [x15, #0x18]\n"
-      "add x12, x12, #0x10\n"
       "ldr d6, [x15, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "ldr x10, [x15, #0x28]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x15, #0x38]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x30]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr d6, [x15, #0x40]\n"
-      "ldr x10, [x15, #0x48]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x50]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x15, #0x60]\n"
-      "ldr x10, [x15, #0x68]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x78]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x78]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x15, #0x80]\n"
-      "ldr x10, [x15, #0x88]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x98]\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x15, #0xa0]\n"
-      "ldr x10, [x15, #0xa8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xb8]\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0xb8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x15, #0xc0]\n"
-      "ldr x10, [x15, #0xc8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xd8]\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x15, #0xe0]\n"
-      "ldr x10, [x15, #0xe8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xf8]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xf0]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0xf8]\n"
+      "mov v7.d[1], x11\n"
+      "add x12, x12, #0x10\n"
       "add x15, x15, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x15, #0x0]\n"
-      "ldr x10, [x15, #0x8]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d0, [x12, #0x0]\n"
-      "mov v0.d[1], x9\n"
+      "sub x13, x13, #0x10\n"
+      "ldr d7, [x15, #0x10]\n"
+      "cmp x13, #0x20\n"
+      "ldr x10, [x12, #0x8]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       "bge 7b\n"
       "8:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
       "ldr q6, [x15, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
+      ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x15, #0x50]\n"
@@ -236,9 +233,12 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "ldr q6, [x15, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       "ldr q7, [x15, #0xf0]\n"
+      "add x12, x12, #0x10\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "add x15, x15, #0x100\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
+      "add x15, x15, #0x100\n"
       "9:"  // Height 1: Multiply loop: Main loop skip
       "cbz x13, 14f\n"
       "cmp x13, #0x4\n"
@@ -247,18 +247,18 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "cmp x13, #0x4\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "ldr q6, [x15, #0x20]\n"
+      "cmp x13, #0x4\n"
       "ldr q7, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x15, x15, #0x40\n"
       "bge 10b\n"
-      "cbz x13, 14f\n"
       "11:"  // Height 1: Multiply loop: Skip odd blocks
+      "cbz x13, 14f\n"
       "tbz x13, #1, 12f\n"
       "ldr h0, [x12], #0x2\n"
       "tbz x13, #0, 13f\n"
@@ -268,46 +268,46 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "ldr b0, [x12, #0x0]\n"
       "13:"  // Height 1: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x15, #0x20]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "ldr q6, [x15, #0x20]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x15, x15, #0x40\n"
       "14:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "ldr q2, [x8, #0x20]\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "ldr q3, [x8, #0x30]\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "add x8, x8, #0x40\n"
+      "ldr q2, [x6, #0x20]\n"
       "add v10.4s, v10.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
       "add v11.4s, v11.4s, v3.4s\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 15f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 16f\n"
       "15:"  // Height 1: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -337,87 +337,87 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v9.4s, v9.4s, v1.4s\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
       "smin v11.4s, v11.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
       "smax v11.4s, v11.4s, v5.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
+      "cmp x16, #0x10\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "bge 26f\n"
-      "tbz x17, #3, 21f\n"
-      "str d8, [x16], #0x8\n"
-      "tbz x17, #2, 19f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "tbz x17, #1, 18f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[14], [x16]\n"
+      "tbz x16, #3, 21f\n"
+      "str d8, [x17], #0x8\n"
+      "tbz x16, #2, 19f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "tbz x16, #1, 18f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[14], [x17]\n"
       "b 25f\n"
       "18:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[12], [x16]\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[12], [x17]\n"
       "b 25f\n"
       "19:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 20f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[10], [x16]\n"
+      "tbz x16, #1, 20f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[10], [x17]\n"
       "b 25f\n"
       "20:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[8], [x16]\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[8], [x17]\n"
       "b 25f\n"
       "21:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 23f\n"
-      "str s8, [x16], #0x4\n"
-      "tbz x17, #1, 22f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[6], [x16]\n"
+      "tbz x16, #2, 23f\n"
+      "str s8, [x17], #0x4\n"
+      "tbz x16, #1, 22f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[6], [x17]\n"
       "b 25f\n"
       "22:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[4], [x16]\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[4], [x17]\n"
       "b 25f\n"
       "23:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 24f\n"
-      "str h8, [x16], #0x2\n"
-      "tbz x17, #0, 25f\n"
-      "st1 { v8.b }[2], [x16]\n"
+      "tbz x16, #1, 24f\n"
+      "str h8, [x17], #0x2\n"
+      "tbz x16, #0, 25f\n"
+      "st1 { v8.b }[2], [x17]\n"
       "b 25f\n"
       "24:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
       "25:"  // Height 1: Partial direct writeback: Done
       "b 27f\n"
       "26:"  // Height 1: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
       "27:"  // Height 1: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 2b\n"
       "b 164f\n"
       "28:"  // Height 2
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x16, %x[output_ptr]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "29:"  // Height 2: Column loop
       "movi v8.4s, #0x0\n"
@@ -432,144 +432,144 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "31:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 32f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
       "cbnz x14, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
+      "add x9, x9, x20\n"
       "b 33f\n"
       "32:"  // Height 2: setup direct input
       "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
+      "add x9, x12, x20\n"
       "33:"  // Height 2: input setup done
       "cmp x13, #0x10\n"
       "blt 36f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
       "cmp x13, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
       "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 35f\n"
       "34:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x18]\n"
       "ldr d6, [x15, #0x20]\n"
-      "add x12, x12, #0x10\n"
-      "ldr x10, [x15, #0x28]\n"
-      "add x28, x28, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "sub x13, x13, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x15, #0x38]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x15, #0x30]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x11, [x15, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       "ldr d6, [x15, #0x40]\n"
-      "ldr x10, [x15, #0x48]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x15, #0x50]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x68]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x15, #0x60]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x78]\n"
-      "ldr x27, [x28, #0x8]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x15, #0x78]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x88]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x15, #0x80]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xa8]\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x15, #0xa0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xc8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x15, #0xc0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x15, #0xe8]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x15, #0xe0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       "ldr d7, [x15, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v6.d[1], x20\n"
+      "add x12, x12, #0x10\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
       "add x15, x15, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x15, #0x0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x10, [x15, #0x8]\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       "ldr d0, [x12, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d1, [x28, #0x0]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
+      "ldr d1, [x9, #0x0]\n"
+      "sub x13, x13, #0x10\n"
+      "ldr d7, [x15, #0x10]\n"
+      "cmp x13, #0x20\n"
+      "ldr x10, [x12, #0x8]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x28, [x9, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x15, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "bge 34b\n"
       "35:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
+      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x15, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr q6, [x15, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
@@ -611,11 +611,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "37:"  // Height 2: Multiply loop: Odd block loop
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s1, [x9], #0x4\n"
       "cmp x13, #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
@@ -627,22 +627,22 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 37b\n"
-      "cbz x13, 41f\n"
       "38:"  // Height 2: Multiply loop: Skip odd blocks
+      "cbz x13, 41f\n"
       "tbz x13, #1, 39f\n"
       "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
       "tbz x13, #0, 40f\n"
       "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
+      "ld1 { v1.b }[2], [x9]\n"
       "b 40f\n"
       "39:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
       "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
       "40:"  // Height 2: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
@@ -654,44 +654,44 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "41:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 31b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "add x23, x16, x19\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "add v13.4s, v13.4s, v1.4s\n"
-      "ldr q2, [x8, #0x20]\n"
-      "ldr q3, [x8, #0x30]\n"
-      "add x8, x8, #0x40\n"
+      "ldr q2, [x6, #0x20]\n"
       "add v10.4s, v10.4s, v2.4s\n"
-      "add v14.4s, v14.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
       "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "add v13.4s, v13.4s, v1.4s\n"
+      "add v14.4s, v14.4s, v2.4s\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 42f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 43f\n"
       "42:"  // Height 2: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -741,13 +741,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v13.4s, v13.4s, v1.4s\n"
       "srshl v14.4s, v14.4s, v2.4s\n"
       "srshl v15.4s, v15.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
@@ -756,6 +751,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v13.4s, v13.4s, v4.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
@@ -764,6 +761,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smin v13.4s, v13.4s, v6.4s\n"
       "smin v14.4s, v14.4s, v6.4s\n"
       "smin v15.4s, v15.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
@@ -776,83 +775,84 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
+      "cmp x16, #0x10\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "uzp1 v12.16b, v12.16b, v13.16b\n"
       "bge 53f\n"
-      "tbz x17, #3, 48f\n"
-      "str d8, [x16], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "tbz x17, #2, 46f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "tbz x17, #1, 45f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[14], [x16]\n"
-      "st1 { v12.b }[14], [x23]\n"
+      "tbz x16, #3, 48f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "tbz x16, #2, 46f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "tbz x16, #1, 45f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[14], [x17]\n"
+      "st1 { v12.b }[14], [x24]\n"
       "b 52f\n"
       "45:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[12], [x16]\n"
-      "st1 { v12.b }[12], [x23]\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[12], [x17]\n"
+      "st1 { v12.b }[12], [x24]\n"
       "b 52f\n"
       "46:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 47f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[10], [x16]\n"
-      "st1 { v12.b }[10], [x23]\n"
+      "tbz x16, #1, 47f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[10], [x17]\n"
+      "st1 { v12.b }[10], [x24]\n"
       "b 52f\n"
       "47:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[8], [x16]\n"
-      "st1 { v12.b }[8], [x23]\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[8], [x17]\n"
+      "st1 { v12.b }[8], [x24]\n"
       "b 52f\n"
       "48:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 50f\n"
-      "str s8, [x16], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "tbz x17, #1, 49f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[6], [x16]\n"
-      "st1 { v12.b }[6], [x23]\n"
+      "tbz x16, #2, 50f\n"
+      "str s8, [x17], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "tbz x16, #1, 49f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[6], [x17]\n"
+      "st1 { v12.b }[6], [x24]\n"
       "b 52f\n"
       "49:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[4], [x16]\n"
-      "st1 { v12.b }[4], [x23]\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[4], [x17]\n"
+      "st1 { v12.b }[4], [x24]\n"
       "b 52f\n"
       "50:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 51f\n"
-      "str h8, [x16], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "tbz x17, #0, 52f\n"
-      "st1 { v8.b }[2], [x16]\n"
-      "st1 { v12.b }[2], [x23]\n"
+      "tbz x16, #1, 51f\n"
+      "str h8, [x17], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "tbz x16, #0, 52f\n"
+      "st1 { v8.b }[2], [x17]\n"
+      "st1 { v12.b }[2], [x24]\n"
       "b 52f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
       "52:"  // Height 2: Partial direct writeback: Done
       "b 54f\n"
       "53:"  // Height 2: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
-      "str q12, [x23, #0x0]\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
+      "str q12, [x24, #0x0]\n"
       "54:"  // Height 2: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 29b\n"
       "b 164f\n"
       "55:"  // Height 3
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x16, %x[output_ptr]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "56:"  // Height 3: Column loop
       "movi v8.4s, #0x0\n"
@@ -871,172 +871,172 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "58:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
       "cbnz x14, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
       "b 60f\n"
       "59:"  // Height 3: setup direct input
       "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
+      "add x9, x12, x20\n"
+      "add x27, x9, x20\n"
       "60:"  // Height 3: input setup done
       "cmp x13, #0x10\n"
       "blt 63f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
       "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
       "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 62f\n"
       "61:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x18]\n"
+      "ldr x11, [x15, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr d6, [x15, #0x20]\n"
-      "ldr x10, [x15, #0x28]\n"
-      "add x12, x12, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x15, #0x38]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr d7, [x15, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x15, #0x48]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x15, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0x40]\n"
-      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x58]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x50]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x11, [x15, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0x60]\n"
-      "cmp x13, #0x20\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x15, #0x78]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x88]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
+      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x15, #0x98]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x15, #0xb8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xc8]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x15, #0xd8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x15, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x15, #0xf8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0xf0]\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
+      "add x27, x27, #0x10\n"
       "add x15, x15, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x15, #0x8]\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
+      "ldr x10, [x12, #0x8]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x15, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       "ldr d0, [x12, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "mov v0.d[1], x9\n"
-      "ldr d2, [x26, #0x0]\n"
-      "mov v1.d[1], x27\n"
-      "mov v2.d[1], x25\n"
+      "ldr d2, [x27, #0x0]\n"
+      "sub x13, x13, #0x10\n"
+      "ldr d7, [x15, #0x10]\n"
+      "cmp x13, #0x20\n"
+      "ldr x26, [x27, #0x8]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x11, [x15, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
+      "mov v2.d[1], x26\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "mov v7.d[1], x11\n"
       "bge 61b\n"
       "62:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x15, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
@@ -1093,12 +1093,12 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "64:"  // Height 3: Multiply loop: Odd block loop
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s1, [x9], #0x4\n"
       "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
@@ -1114,25 +1114,25 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 64b\n"
-      "cbz x13, 68f\n"
       "65:"  // Height 3: Multiply loop: Skip odd blocks
+      "cbz x13, 68f\n"
       "tbz x13, #1, 66f\n"
       "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
       "tbz x13, #0, 67f\n"
       "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
       "b 67f\n"
       "66:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
       "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
       "67:"  // Height 3: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
@@ -1148,50 +1148,50 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "68:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "add x23, x16, x19\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "add v13.4s, v13.4s, v1.4s\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "ldr q2, [x8, #0x20]\n"
-      "ldr q3, [x8, #0x30]\n"
-      "add x8, x8, #0x40\n"
+      "ldr q2, [x6, #0x20]\n"
       "add v10.4s, v10.4s, v2.4s\n"
-      "add v14.4s, v14.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
       "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "add v13.4s, v13.4s, v1.4s\n"
+      "add v14.4s, v14.4s, v2.4s\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 69f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 70f\n"
       "69:"  // Height 3: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -1221,10 +1221,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
-      "and v4.16b, v12.16b, v0.16b\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
+      "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
@@ -1233,11 +1233,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v12.4s, v12.4s, v4.4s\n"
-      "and v4.16b, v16.16b, v0.16b\n"
       "sqadd v13.4s, v13.4s, v5.4s\n"
-      "and v5.16b, v17.16b, v1.16b\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
+      "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
@@ -1261,13 +1261,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v17.4s, v17.4s, v1.4s\n"
       "srshl v18.4s, v18.4s, v2.4s\n"
       "srshl v19.4s, v19.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
@@ -1278,6 +1273,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
@@ -1288,6 +1287,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smin v15.4s, v15.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
@@ -1298,112 +1301,109 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smax v15.4s, v15.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "add v18.4s, v18.4s, v4.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "cmp x16, #0x10\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "uzp1 v12.16b, v12.16b, v13.16b\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 80f\n"
-      "tbz x17, #3, 75f\n"
-      "str d8, [x16], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x17, #2, 73f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "tbz x17, #1, 72f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[14], [x16]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
+      "tbz x16, #3, 75f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x16, #2, 73f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "tbz x16, #1, 72f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[14], [x17]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
       "b 79f\n"
       "72:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[12], [x16]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[12], [x17]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
       "b 79f\n"
       "73:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 74f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[10], [x16]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
+      "tbz x16, #1, 74f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[10], [x17]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
       "b 79f\n"
       "74:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[8], [x16]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[8], [x17]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
       "b 79f\n"
       "75:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 77f\n"
-      "str s8, [x16], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "tbz x17, #1, 76f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[6], [x16]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
+      "tbz x16, #2, 77f\n"
+      "str s8, [x17], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "tbz x16, #1, 76f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[6], [x17]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
       "b 79f\n"
       "76:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[4], [x16]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[4], [x17]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
       "b 79f\n"
       "77:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 78f\n"
-      "str h8, [x16], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "tbz x17, #0, 79f\n"
-      "st1 { v8.b }[2], [x16]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
+      "tbz x16, #1, 78f\n"
+      "str h8, [x17], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "tbz x16, #0, 79f\n"
+      "st1 { v8.b }[2], [x17]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
       "b 79f\n"
       "78:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
       "79:"  // Height 3: Partial direct writeback: Done
       "b 81f\n"
       "80:"  // Height 3: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
       "81:"  // Height 3: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 56b\n"
       "b 164f\n"
       "82:"  // Height 4
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x16, %x[output_ptr]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "83:"  // Height 4: Column loop
       "movi v8.4s, #0x0\n"
@@ -1426,201 +1426,201 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "85:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 86f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
       "cbnz x14, 87f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
       "b 87f\n"
       "86:"  // Height 4: setup direct input
       "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
+      "add x9, x12, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
       "87:"  // Height 4: input setup done
       "cmp x13, #0x10\n"
       "blt 90f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
       "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 89f\n"
       "88:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x18]\n"
+      "ldr x11, [x15, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x15, #0x28]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x15, #0x20]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x15, #0x38]\n"
-      "add x12, x12, #0x10\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x15, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x15, #0x48]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x15, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x11, [x15, #0x58]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x15, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "add x26, x26, #0x10\n"
+      "ldr x11, [x15, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x12, #0x8]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       "ldr d6, [x15, #0x60]\n"
+      ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x11, [x15, #0x78]\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x70]\n"
-      "add x24, x24, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x10, [x15, #0x88]\n"
+      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       "ldr d6, [x15, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x15, #0x98]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "sub x13, x13, #0x10\n"
+      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x15, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x15, #0xb8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "cmp x13, #0x20\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "cmp x13, #0x20\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x15, #0xc8]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x15, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x15, #0xd8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x15, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x15, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x15, #0xf8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       "ldr d7, [x15, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x15, x15, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x15, #0x8]\n"
+      "ldr x11, [x15, #0x18]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x15, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       "ldr d0, [x12, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
+      "ldr d3, [x25, #0x0]\n"
+      "ldr d7, [x15, #0x10]\n"
+      "mov v6.d[1], x20\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v7.d[1], x11\n"
       "bge 88b\n"
       "89:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
+      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
@@ -1691,13 +1691,13 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "91:"  // Height 4: Multiply loop: Odd block loop
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s1, [x9], #0x4\n"
       "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1717,28 +1717,28 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 91b\n"
-      "cbz x13, 95f\n"
       "92:"  // Height 4: Multiply loop: Skip odd blocks
+      "cbz x13, 95f\n"
       "tbz x13, #1, 93f\n"
       "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
       "tbz x13, #0, 94f\n"
       "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
       "b 94f\n"
       "93:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
       "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
       "94:"  // Height 4: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1758,56 +1758,56 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "95:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 85b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "add x23, x16, x19\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "add v13.4s, v13.4s, v1.4s\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "ldr q2, [x8, #0x20]\n"
-      "ldr q3, [x8, #0x30]\n"
-      "add x8, x8, #0x40\n"
+      "ldr q2, [x6, #0x20]\n"
       "add v10.4s, v10.4s, v2.4s\n"
-      "add v14.4s, v14.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
       "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "add v13.4s, v13.4s, v1.4s\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "add v14.4s, v14.4s, v2.4s\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 96f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 97f\n"
       "96:"  // Height 4: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -1841,10 +1841,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
-      "and v4.16b, v12.16b, v0.16b\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
+      "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
@@ -1853,11 +1853,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v12.4s, v12.4s, v4.4s\n"
-      "and v4.16b, v16.16b, v0.16b\n"
       "sqadd v13.4s, v13.4s, v5.4s\n"
-      "and v5.16b, v17.16b, v1.16b\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
+      "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
@@ -1865,18 +1865,18 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v20.16b, v0.16b\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
-      "and v5.16b, v21.16b, v1.16b\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "and v4.16b, v20.16b, v0.16b\n"
+      "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
       "and v7.16b, v23.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
       "sqadd v23.4s, v23.4s, v7.4s\n"
@@ -1897,13 +1897,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v21.4s, v21.4s, v1.4s\n"
       "srshl v22.4s, v22.4s, v2.4s\n"
       "srshl v23.4s, v23.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
@@ -1914,6 +1909,14 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "add v20.4s, v20.4s, v4.4s\n"
+      "add v21.4s, v21.4s, v4.4s\n"
+      "add v22.4s, v22.4s, v4.4s\n"
+      "add v23.4s, v23.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
@@ -1924,6 +1927,14 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smin v15.4s, v15.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
@@ -1934,143 +1945,132 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smax v15.4s, v15.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "add v18.4s, v18.4s, v4.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
-      "add v20.4s, v20.4s, v4.4s\n"
-      "add v21.4s, v21.4s, v4.4s\n"
-      "add v22.4s, v22.4s, v4.4s\n"
-      "add v23.4s, v23.4s, v4.4s\n"
-      "uzp1 v8.8h, v8.8h, v9.8h\n"
-      "uzp1 v9.8h, v10.8h, v11.8h\n"
-      "uzp1 v12.8h, v12.8h, v13.8h\n"
-      "uzp1 v13.8h, v14.8h, v15.8h\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "uzp1 v8.16b, v8.16b, v9.16b\n"
-      "uzp1 v12.16b, v12.16b, v13.16b\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
       "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
+      "uzp1 v8.8h, v8.8h, v9.8h\n"
+      "uzp1 v9.8h, v10.8h, v11.8h\n"
+      "uzp1 v12.8h, v12.8h, v13.8h\n"
+      "uzp1 v13.8h, v14.8h, v15.8h\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "cmp x16, #0x10\n"
+      "uzp1 v8.16b, v8.16b, v9.16b\n"
+      "uzp1 v12.16b, v12.16b, v13.16b\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 107f\n"
-      "tbz x17, #3, 102f\n"
-      "str d8, [x16], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x17, #2, 100f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "tbz x17, #1, 99f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[14], [x16]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "tbz x16, #3, 102f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x16, #2, 100f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "tbz x16, #1, 99f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[14], [x17]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 106f\n"
       "99:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[12], [x16]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[12], [x17]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 106f\n"
       "100:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 101f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[10], [x16]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "tbz x16, #1, 101f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[10], [x17]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 106f\n"
       "101:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[8], [x16]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[8], [x17]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 106f\n"
       "102:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 104f\n"
-      "str s8, [x16], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "tbz x17, #1, 103f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[6], [x16]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "tbz x16, #2, 104f\n"
+      "str s8, [x17], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "tbz x16, #1, 103f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[6], [x17]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 106f\n"
       "103:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[4], [x16]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[4], [x17]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 106f\n"
       "104:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 105f\n"
-      "str h8, [x16], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "tbz x17, #0, 106f\n"
-      "st1 { v8.b }[2], [x16]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "tbz x16, #1, 105f\n"
+      "str h8, [x17], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "tbz x16, #0, 106f\n"
+      "st1 { v8.b }[2], [x17]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 106f\n"
       "105:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "106:"  // Height 4: Partial direct writeback: Done
       "b 108f\n"
       "107:"  // Height 4: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
       "108:"  // Height 4: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 83b\n"
       "b 164f\n"
       "109:"  // Height 5
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x16, %x[output_ptr]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "110:"  // Height 5: Column loop
       "movi v8.4s, #0x0\n"
@@ -2097,228 +2097,228 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "112:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 113f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
       "cbnz x14, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
       "b 114f\n"
       "113:"  // Height 5: setup direct input
       "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
+      "add x9, x12, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
       "114:"  // Height 5: input setup done
       "cmp x13, #0x10\n"
       "blt 117f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
       "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
       "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 116f\n"
       "115:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x18]\n"
+      "ldr x11, [x15, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x15, #0x28]\n"
-      ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       "add x12, x12, #0x10\n"
+      ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "add x28, x28, #0x10\n"
+      "ldr d6, [x15, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr d6, [x15, #0x20]\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x11, [x15, #0x38]\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x15, #0x30]\n"
-      "add x26, x26, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x10, [x15, #0x48]\n"
+      "ldr x11, [x15, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x15, #0x58]\n"
+      "ldr x10, [x12, #0x8]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr d6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x15, #0x68]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "ldr d7, [x15, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x15, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x15, #0x78]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x15, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "cmp x13, #0x20\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x15, #0x88]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x15, #0x98]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x15, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x15, #0xa8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x15, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x15, #0xc8]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x15, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x15, #0xe8]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x15, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       "ldr d7, [x15, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x15, x15, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x15, #0x8]\n"
+      "ldr x11, [x15, #0x18]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x15, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       "ldr d0, [x12, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
+      "ldr d4, [x23, #0x0]\n"
+      "ldr d7, [x15, #0x10]\n"
+      "mov v6.d[1], x20\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v7.d[1], x11\n"
       "bge 115b\n"
       "116:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
@@ -2405,14 +2405,14 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "118:"  // Height 5: Multiply loop: Odd block loop
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s1, [x9], #0x4\n"
       "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2436,31 +2436,31 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 118b\n"
-      "cbz x13, 122f\n"
       "119:"  // Height 5: Multiply loop: Skip odd blocks
+      "cbz x13, 122f\n"
       "tbz x13, #1, 120f\n"
       "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
       "tbz x13, #0, 121f\n"
       "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 121f\n"
       "120:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
       "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "121:"  // Height 5: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2484,62 +2484,62 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "122:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 112b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "add x23, x16, x19\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
+      "ldr q2, [x6, #0x20]\n"
+      "add v10.4s, v10.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "add v13.4s, v13.4s, v1.4s\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
-      "add v24.4s, v24.4s, v0.4s\n"
-      "add v25.4s, v25.4s, v1.4s\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "ldr q2, [x8, #0x20]\n"
-      "ldr q3, [x8, #0x30]\n"
-      "add x8, x8, #0x40\n"
-      "add v10.4s, v10.4s, v2.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add v24.4s, v24.4s, v0.4s\n"
+      "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 123f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 124f\n"
       "123:"  // Height 5: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -2577,10 +2577,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
-      "and v4.16b, v12.16b, v0.16b\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
+      "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
@@ -2589,11 +2589,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v12.4s, v12.4s, v4.4s\n"
-      "and v4.16b, v16.16b, v0.16b\n"
       "sqadd v13.4s, v13.4s, v5.4s\n"
-      "and v5.16b, v17.16b, v1.16b\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
+      "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
@@ -2601,31 +2601,31 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v20.16b, v0.16b\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
-      "and v5.16b, v21.16b, v1.16b\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "and v4.16b, v20.16b, v0.16b\n"
+      "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
       "and v7.16b, v23.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "and v4.16b, v24.16b, v0.16b\n"
+      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
-      "and v5.16b, v25.16b, v1.16b\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
-      "and v6.16b, v26.16b, v2.16b\n"
       "sqadd v23.4s, v23.4s, v7.4s\n"
+      "and v4.16b, v24.16b, v0.16b\n"
+      "and v5.16b, v25.16b, v1.16b\n"
+      "and v6.16b, v26.16b, v2.16b\n"
+      "and v7.16b, v27.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "and v7.16b, v27.16b, v3.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v24.4s, v24.4s, v4.4s\n"
       "sqadd v25.4s, v25.4s, v5.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
       "125:"  // Height 5: no shift correction
@@ -2649,13 +2649,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v25.4s, v25.4s, v1.4s\n"
       "srshl v26.4s, v26.4s, v2.4s\n"
       "srshl v27.4s, v27.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
@@ -2666,26 +2661,6 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
@@ -2696,6 +2671,18 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
       "add v27.4s, v27.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v20.4s, v20.4s, v6.4s\n"
@@ -2706,6 +2693,18 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smin v25.4s, v25.4s, v6.4s\n"
       "smin v26.4s, v26.4s, v6.4s\n"
       "smin v27.4s, v27.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
@@ -2726,138 +2725,139 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
+      "cmp x16, #0x10\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "uzp1 v12.16b, v12.16b, v13.16b\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 134f\n"
-      "tbz x17, #3, 129f\n"
-      "str d8, [x16], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x17, #2, 127f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "tbz x17, #1, 126f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[14], [x16]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "tbz x16, #3, 129f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x16, #2, 127f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "tbz x16, #1, 126f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[14], [x17]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 133f\n"
       "126:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[12], [x16]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[12], [x17]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 133f\n"
       "127:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 128f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[10], [x16]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "tbz x16, #1, 128f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[10], [x17]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 133f\n"
       "128:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[8], [x16]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[8], [x17]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 133f\n"
       "129:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 131f\n"
-      "str s8, [x16], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "tbz x17, #1, 130f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[6], [x16]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "tbz x16, #2, 131f\n"
+      "str s8, [x17], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "tbz x16, #1, 130f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[6], [x17]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 133f\n"
       "130:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[4], [x16]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[4], [x17]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 133f\n"
       "131:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 132f\n"
-      "str h8, [x16], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "tbz x17, #0, 133f\n"
-      "st1 { v8.b }[2], [x16]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "tbz x16, #1, 132f\n"
+      "str h8, [x17], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "tbz x16, #0, 133f\n"
+      "st1 { v8.b }[2], [x17]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 133f\n"
       "132:"  // Height 5: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "133:"  // Height 5: Partial direct writeback: Done
       "b 135f\n"
       "134:"  // Height 5: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "135:"  // Height 5: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 110b\n"
       "b 164f\n"
       "136:"  // Height 6
-      "ldr x6, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x8, %x[col_bias]\n"
-      "ldr x7, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x16, %x[output_ptr]\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x6\n"
+      "mov x6, %x[col_bias]\n"
+      "ldr x7, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
       "137:"  // Height 6: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -2887,257 +2887,257 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "mov x14, #0x0\n"
       "139:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 140f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
+      "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x12, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
       "cbnz x14, 141f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x12, x12, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
+      "add x21, x21, x20\n"
       "b 141f\n"
       "140:"  // Height 6: setup direct input
       "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
-      "add x20, x22, x19\n"
+      "add x9, x12, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
+      "add x21, x23, x20\n"
       "141:"  // Height 6: input setup done
       "cmp x13, #0x10\n"
       "blt 144f\n"
       "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
       "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x15, #0x0]\n"
+      "ldr q7, [x15, #0x10]\n"
       "blt 143f\n"
       "142:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x15, #0x10]\n"
+      "ldr x20, [x15, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x18]\n"
+      "ldr x11, [x15, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x15, #0x28]\n"
-      ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       "add x12, x12, #0x10\n"
+      ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr d6, [x15, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x15, #0x38]\n"
+      "ldr x20, [x15, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr x10, [x15, #0x48]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x15, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x11, [x15, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x12, #0x8]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x15, #0x58]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr d6, [x15, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "ldr x20, [x15, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x15, #0x68]\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "cmp x13, #0x20\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x15, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x15, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x15, #0x78]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x15, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr x20, [x15, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x15, #0x88]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x15, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x15, #0x98]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "sub x13, x13, #0x10\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x15, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
+      "ldr x20, [x15, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x15, #0xa8]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x15, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x15, #0xb8]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x15, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x15, #0xc8]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x15, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x15, #0xd8]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x15, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x20, [x15, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x15, #0xe8]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x15, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x15, #0xf8]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x15, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x20\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr x19, [x20, #0x8]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
       "ldr d7, [x15, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x15, x15, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x20, [x15, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
-      "ldr x10, [x15, #0x8]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8de  // sdot v30.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x15, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       "ldr d0, [x12, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
+      "ldr d4, [x23, #0x0]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "ldr d5, [x20, #0x0]\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
-      "mov v5.d[1], x19\n"
+      "ldr d5, [x21, #0x0]\n"
+      "ldr d7, [x15, #0x10]\n"
+      "mov v6.d[1], x20\n"
+      "ldr x20, [x21, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x15, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v5.d[1], x20\n"
+      "mov v7.d[1], x11\n"
       "bge 142b\n"
       "143:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x15, #0x10]\n"
+      "add x12, x12, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x15, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "sub x13, x13, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x12, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr q7, [x15, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x20, x20, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr q6, [x15, #0x40]\n"
@@ -3238,15 +3238,15 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "145:"  // Height 6: Multiply loop: Odd block loop
       "ldr s0, [x12], #0x4\n"
       "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
+      "ldr s1, [x9], #0x4\n"
       "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3274,34 +3274,34 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 145b\n"
-      "cbz x13, 149f\n"
       "146:"  // Height 6: Multiply loop: Skip odd blocks
+      "cbz x13, 149f\n"
       "tbz x13, #1, 147f\n"
       "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
       "tbz x13, #0, 148f\n"
       "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 148f\n"
       "147:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
       "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "148:"  // Height 6: Multiply loop: Ragged operand read: Done
       "ldr q6, [x15, #0x0]\n"
-      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x15, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3329,68 +3329,68 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "149:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "cmp x14, x20\n"
       "bne 139b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x16, #0x0]\n"
-      "ldr q0, [x8, #0x0]\n"
-      "ldr q1, [x8, #0x10]\n"
-      "add x23, x16, x19\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr q0, [x6, #0x0]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "ldr q1, [x6, #0x10]\n"
       "add v9.4s, v9.4s, v1.4s\n"
+      "ldr q2, [x6, #0x20]\n"
+      "add v10.4s, v10.4s, v2.4s\n"
+      "ldr q3, [x6, #0x30]\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "add v13.4s, v13.4s, v1.4s\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
-      "add v24.4s, v24.4s, v0.4s\n"
-      "add v25.4s, v25.4s, v1.4s\n"
-      "add v28.4s, v28.4s, v0.4s\n"
-      "add v29.4s, v29.4s, v1.4s\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "add v14.4s, v14.4s, v2.4s\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
+      "add v15.4s, v15.4s, v3.4s\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
+      "add v16.4s, v16.4s, v0.4s\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
-      "ldr q2, [x8, #0x20]\n"
-      "ldr q3, [x8, #0x30]\n"
-      "add x8, x8, #0x40\n"
-      "add v10.4s, v10.4s, v2.4s\n"
-      "add v14.4s, v14.4s, v2.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
-      "add v15.4s, v15.4s, v3.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add v24.4s, v24.4s, v0.4s\n"
+      "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
+      "add v28.4s, v28.4s, v0.4s\n"
+      "add v29.4s, v29.4s, v1.4s\n"
       "add v30.4s, v30.4s, v2.4s\n"
       "add v31.4s, v31.4s, v3.4s\n"
+      "add x6, x6, #0x40\n"
       "tbz %x[flags], #4, 150f\n"
-      "ldr q0, [x7, #0x0]\n"
-      "ldr q4, [x6, #0x0]\n"
-      "ldr q1, [x7, #0x10]\n"
-      "ldr q5, [x6, #0x10]\n"
-      "ldr q2, [x7, #0x20]\n"
-      "ldr q6, [x6, #0x20]\n"
-      "ldr q3, [x7, #0x30]\n"
+      "ldr q0, [x8, #0x0]\n"
+      "ldr q4, [x7, #0x0]\n"
+      "ldr q1, [x8, #0x10]\n"
+      "ldr q5, [x7, #0x10]\n"
+      "ldr q2, [x8, #0x20]\n"
+      "ldr q6, [x7, #0x20]\n"
+      "ldr q3, [x8, #0x30]\n"
+      "add x8, x8, #0x40\n"
+      "ldr q7, [x7, #0x30]\n"
       "add x7, x7, #0x40\n"
-      "ldr q7, [x6, #0x30]\n"
-      "add x6, x6, #0x40\n"
       "b 151f\n"
       "150:"  // Height 6: per layer parameters
       "add x25, %x[qp], %[per_layer_right_shift]\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
       "ld1r { v0.4s }, [x25]\n"
-      "ld1r { v4.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
       "mov v2.16b, v0.16b\n"
@@ -3432,10 +3432,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
-      "and v4.16b, v12.16b, v0.16b\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
+      "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
@@ -3444,11 +3444,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v12.4s, v12.4s, v4.4s\n"
-      "and v4.16b, v16.16b, v0.16b\n"
       "sqadd v13.4s, v13.4s, v5.4s\n"
-      "and v5.16b, v17.16b, v1.16b\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
+      "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
@@ -3456,42 +3456,42 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v20.16b, v0.16b\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
-      "and v5.16b, v21.16b, v1.16b\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "and v4.16b, v20.16b, v0.16b\n"
+      "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
       "and v7.16b, v23.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "and v4.16b, v24.16b, v0.16b\n"
+      "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
-      "and v5.16b, v25.16b, v1.16b\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
-      "and v6.16b, v26.16b, v2.16b\n"
       "sqadd v23.4s, v23.4s, v7.4s\n"
+      "and v4.16b, v24.16b, v0.16b\n"
+      "and v5.16b, v25.16b, v1.16b\n"
+      "and v6.16b, v26.16b, v2.16b\n"
+      "and v7.16b, v27.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "and v7.16b, v27.16b, v3.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v24.4s, v24.4s, v4.4s\n"
-      "and v4.16b, v28.16b, v0.16b\n"
       "sqadd v25.4s, v25.4s, v5.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "and v5.16b, v29.16b, v1.16b\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
-      "and v6.16b, v30.16b, v2.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
+      "and v4.16b, v28.16b, v0.16b\n"
+      "and v5.16b, v29.16b, v1.16b\n"
+      "and v6.16b, v30.16b, v2.16b\n"
       "and v7.16b, v31.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
-      "sqadd v28.4s, v28.4s, v4.4s\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v28.4s, v28.4s, v4.4s\n"
       "sqadd v29.4s, v29.4s, v5.4s\n"
       "sqadd v30.4s, v30.4s, v6.4s\n"
       "sqadd v31.4s, v31.4s, v7.4s\n"
@@ -3520,13 +3520,8 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "srshl v29.4s, v29.4s, v1.4s\n"
       "srshl v30.4s, v30.4s, v2.4s\n"
       "srshl v31.4s, v31.4s, v3.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "add x25, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x25]\n"
-      "cmp x17, #0x10\n"
-      "ld1r { v6.4s }, [x24]\n"
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
@@ -3537,26 +3532,6 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
@@ -3567,6 +3542,22 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
       "add v27.4s, v27.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v20.4s, v20.4s, v6.4s\n"
@@ -3577,6 +3568,22 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smin v25.4s, v25.4s, v6.4s\n"
       "smin v26.4s, v26.4s, v6.4s\n"
       "smin v27.4s, v27.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
@@ -3587,184 +3594,177 @@ void a64_hybrid_s8qs_dot_6x16_a55 (
       "smax v25.4s, v25.4s, v5.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
+      "smax v28.4s, v28.4s, v5.4s\n"
+      "smax v29.4s, v29.4s, v5.4s\n"
+      "smax v30.4s, v30.4s, v5.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
+      "uzp1 v28.8h, v28.8h, v29.8h\n"
+      "uzp1 v29.8h, v30.8h, v31.8h\n"
+      "cmp x16, #0x10\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "uzp1 v12.16b, v12.16b, v13.16b\n"
-      "smax v28.4s, v28.4s, v5.4s\n"
-      "smax v29.4s, v29.4s, v5.4s\n"
-      "smax v30.4s, v30.4s, v5.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
-      "uzp1 v28.8h, v28.8h, v29.8h\n"
-      "uzp1 v29.8h, v30.8h, v31.8h\n"
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 161f\n"
-      "tbz x17, #3, 156f\n"
-      "str d8, [x16], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x17, #2, 154f\n"
-      "st1 { v8.s }[2], [x16], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
-      "tbz x17, #1, 153f\n"
-      "st1 { v8.h }[6], [x16], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[14], [x16]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "tbz x16, #3, 156f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x16, #2, 154f\n"
+      "st1 { v8.s }[2], [x17], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
+      "tbz x16, #1, 153f\n"
+      "st1 { v8.h }[6], [x17], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[14], [x17]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 160f\n"
       "153:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[12], [x16]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[12], [x17]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 160f\n"
       "154:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 155f\n"
-      "st1 { v8.h }[4], [x16], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[10], [x16]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "tbz x16, #1, 155f\n"
+      "st1 { v8.h }[4], [x17], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[10], [x17]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 160f\n"
       "155:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[8], [x16]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[8], [x17]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 160f\n"
       "156:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 158f\n"
-      "str s8, [x16], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
-      "tbz x17, #1, 157f\n"
-      "st1 { v8.h }[2], [x16], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[6], [x16]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "tbz x16, #2, 158f\n"
+      "str s8, [x17], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
+      "tbz x16, #1, 157f\n"
+      "st1 { v8.h }[2], [x17], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[6], [x17]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 160f\n"
       "157:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[4], [x16]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[4], [x17]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 160f\n"
       "158:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 159f\n"
-      "str h8, [x16], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
-      "tbz x17, #0, 160f\n"
-      "st1 { v8.b }[2], [x16]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "tbz x16, #1, 159f\n"
+      "str h8, [x17], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
+      "tbz x16, #0, 160f\n"
+      "st1 { v8.b }[2], [x17]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 160f\n"
       "159:"  // Height 6: Partial direct writeback: partial_1_0
-      "str b8, [x16, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b8, [x17, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "160:"  // Height 6: Partial direct writeback: Done
       "b 162f\n"
       "161:"  // Height 6: Full writeback
-      "str q8, [x16, #0x0]\n"
-      "add x16, x16, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q8, [x17, #0x0]\n"
+      "add x17, x17, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "162:"  // Height 6: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x16, x16, #0x10\n"
       "bgt 137b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 164f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 163f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "163:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "164:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp
index f503f40b0c..598d1524e8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -95,168 +95,168 @@ void a64_hybrid_s8qs_dot_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 55f\n"
       "beq 28f\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[output_ptr]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 9f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q6, [x28, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 8f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "add x9, x9, #0x100\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 7b\n"
       "8:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x9, x9, #0x100\n"
       "9:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 14f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 14f\n"
+      "cmp x27, #0x4\n"
       "blt 11f\n"
       "10:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "cmp x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "ldr q7, [x9, #0x10]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "cmp x27, #0x4\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x9, x9, #0x40\n"
       "bge 10b\n"
-      "cbz x26, 14f\n"
       "11:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 12f\n"
-      "ldr h0, [x25], #0x2\n"
-      "tbz x26, #0, 13f\n"
-      "ld1 { v0.b }[2], [x25]\n"
+      "cbz x27, 14f\n"
+      "tbz x27, #1, 12f\n"
+      "ldr h0, [x26], #0x2\n"
+      "tbz x27, #0, 13f\n"
+      "ld1 { v0.b }[2], [x26]\n"
       "b 13f\n"
       "12:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
       "13:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q6, [x9, #0x20]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x9, x9, #0x40\n"
       "14:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "add v10.4s, v10.4s, v2.4s\n"
       "add v11.4s, v11.4s, v3.4s\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x14, x14, #0x40\n"
       "tbz %x[flags], #4, 15f\n"
       "ldr q0, [x12, #0x0]\n"
       "ldr q4, [x13, #0x0]\n"
@@ -265,20 +265,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 16f\n"
       "15:"  // Height 1: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "16:"  // Height 1: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -293,98 +293,98 @@ void a64_hybrid_s8qs_dot_6x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
       "17:"  // Height 1: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
+      "cmp x10, #0x10\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "uzp1 v8.8h, v8.8h, v9.8h\n"
       "smax v11.4s, v11.4s, v5.4s\n"
+      "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "bge 26f\n"
       "tbz x10, #3, 21f\n"
-      "str d8, [x9], #0x8\n"
+      "str d8, [x11], #0x8\n"
       "tbz x10, #2, 19f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
       "tbz x10, #1, 18f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[14], [x9]\n"
+      "st1 { v8.b }[14], [x11]\n"
       "b 25f\n"
       "18:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[12], [x9]\n"
+      "st1 { v8.b }[12], [x11]\n"
       "b 25f\n"
       "19:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x10, #1, 20f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[10], [x9]\n"
+      "st1 { v8.b }[10], [x11]\n"
       "b 25f\n"
       "20:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[8], [x9]\n"
+      "st1 { v8.b }[8], [x11]\n"
       "b 25f\n"
       "21:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x10, #2, 23f\n"
-      "str s8, [x9], #0x4\n"
+      "str s8, [x11], #0x4\n"
       "tbz x10, #1, 22f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[6], [x9]\n"
+      "st1 { v8.b }[6], [x11]\n"
       "b 25f\n"
       "22:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[4], [x9]\n"
+      "st1 { v8.b }[4], [x11]\n"
       "b 25f\n"
       "23:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x10, #1, 24f\n"
-      "str h8, [x9], #0x2\n"
+      "str h8, [x11], #0x2\n"
       "tbz x10, #0, 25f\n"
-      "st1 { v8.b }[2], [x9]\n"
+      "st1 { v8.b }[2], [x11]\n"
       "b 25f\n"
       "24:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
       "25:"  // Height 1: Partial direct writeback: Done
       "b 27f\n"
       "26:"  // Height 1: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
       "27:"  // Height 1: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 2b\n"
       "b 164f\n"
       "28:"  // Height 2
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "29:"  // Height 2: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -395,215 +395,215 @@ void a64_hybrid_s8qs_dot_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "30:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "31:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 32f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 33f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 33f\n"
       "32:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "33:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 36f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 35f\n"
       "34:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "bge 34b\n"
       "35:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       "36:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 41f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 41f\n"
+      "cmp x27, #0x4\n"
       "blt 38f\n"
       "37:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 37b\n"
-      "cbz x26, 41f\n"
       "38:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 39f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "tbz x26, #0, 40f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
+      "cbz x27, 41f\n"
+      "tbz x27, #1, 39f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "tbz x27, #0, 40f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
       "b 40f\n"
       "39:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
       "40:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "41:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 31b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add v12.4s, v12.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "add v10.4s, v10.4s, v2.4s\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "add v13.4s, v13.4s, v1.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
+      "add x14, x14, #0x40\n"
       "add v15.4s, v15.4s, v3.4s\n"
       "tbz %x[flags], #4, 42f\n"
       "ldr q0, [x12, #0x0]\n"
@@ -613,20 +613,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 43f\n"
       "42:"  // Height 2: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "43:"  // Height 2: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -641,149 +641,149 @@ void a64_hybrid_s8qs_dot_6x16 (
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
       "and v6.16b, v10.16b, v2.16b\n"
+      "and v7.16b, v11.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "and v7.16b, v11.16b, v3.16b\n"
+      "sqadd v11.4s, v11.4s, v7.4s\n"
       "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v12.4s, v12.4s, v4.4s\n"
-      "sqadd v13.4s, v13.4s, v5.4s\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v12.4s, v12.4s, v4.4s\n"
+      "sqadd v13.4s, v13.4s, v5.4s\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
       "44:"  // Height 2: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v12.4s, v12.4s, v0.4s\n"
       "srshl v13.4s, v13.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v14.4s, v14.4s, v2.4s\n"
+      "srshl v15.4s, v15.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
+      "add v14.4s, v14.4s, v4.4s\n"
+      "add v15.4s, v15.4s, v4.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
       "smin v11.4s, v11.4s, v6.4s\n"
       "smin v12.4s, v12.4s, v6.4s\n"
       "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
       "smax v11.4s, v11.4s, v5.4s\n"
       "smax v12.4s, v12.4s, v5.4s\n"
       "smax v13.4s, v13.4s, v5.4s\n"
-      "srshl v14.4s, v14.4s, v2.4s\n"
-      "srshl v15.4s, v15.4s, v3.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
-      "add v14.4s, v14.4s, v4.4s\n"
-      "add v15.4s, v15.4s, v4.4s\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "uzp1 v8.16b, v8.16b, v9.16b\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
+      "uzp1 v8.16b, v8.16b, v9.16b\n"
       "uzp1 v12.16b, v12.16b, v13.16b\n"
       "bge 53f\n"
       "tbz x10, #3, 48f\n"
-      "str d8, [x9], #0x8\n"
-      "str d12, [x23], #0x8\n"
+      "str d8, [x11], #0x8\n"
+      "str d12, [x24], #0x8\n"
       "tbz x10, #2, 46f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
       "tbz x10, #1, 45f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[14], [x9]\n"
-      "st1 { v12.b }[14], [x23]\n"
+      "st1 { v8.b }[14], [x11]\n"
+      "st1 { v12.b }[14], [x24]\n"
       "b 52f\n"
       "45:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[12], [x9]\n"
-      "st1 { v12.b }[12], [x23]\n"
+      "st1 { v8.b }[12], [x11]\n"
+      "st1 { v12.b }[12], [x24]\n"
       "b 52f\n"
       "46:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x10, #1, 47f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[10], [x9]\n"
-      "st1 { v12.b }[10], [x23]\n"
+      "st1 { v8.b }[10], [x11]\n"
+      "st1 { v12.b }[10], [x24]\n"
       "b 52f\n"
       "47:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[8], [x9]\n"
-      "st1 { v12.b }[8], [x23]\n"
+      "st1 { v8.b }[8], [x11]\n"
+      "st1 { v12.b }[8], [x24]\n"
       "b 52f\n"
       "48:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x10, #2, 50f\n"
-      "str s8, [x9], #0x4\n"
-      "str s12, [x23], #0x4\n"
+      "str s8, [x11], #0x4\n"
+      "str s12, [x24], #0x4\n"
       "tbz x10, #1, 49f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[6], [x9]\n"
-      "st1 { v12.b }[6], [x23]\n"
+      "st1 { v8.b }[6], [x11]\n"
+      "st1 { v12.b }[6], [x24]\n"
       "b 52f\n"
       "49:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[4], [x9]\n"
-      "st1 { v12.b }[4], [x23]\n"
+      "st1 { v8.b }[4], [x11]\n"
+      "st1 { v12.b }[4], [x24]\n"
       "b 52f\n"
       "50:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x10, #1, 51f\n"
-      "str h8, [x9], #0x2\n"
-      "str h12, [x23], #0x2\n"
+      "str h8, [x11], #0x2\n"
+      "str h12, [x24], #0x2\n"
       "tbz x10, #0, 52f\n"
-      "st1 { v8.b }[2], [x9]\n"
-      "st1 { v12.b }[2], [x23]\n"
+      "st1 { v8.b }[2], [x11]\n"
+      "st1 { v12.b }[2], [x24]\n"
       "b 52f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
       "52:"  // Height 2: Partial direct writeback: Done
       "b 54f\n"
       "53:"  // Height 2: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q12, [x23, #0x0]\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q12, [x24, #0x0]\n"
       "54:"  // Height 2: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 29b\n"
       "b 164f\n"
       "55:"  // Height 3
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "56:"  // Height 3: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -798,180 +798,180 @@ void a64_hybrid_s8qs_dot_6x16 (
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "57:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 60f\n"
       "59:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "60:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 63f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 62f\n"
       "61:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "bge 61b\n"
       "62:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -979,26 +979,26 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       "63:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 68f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 68f\n"
+      "cmp x27, #0x4\n"
       "blt 65f\n"
       "64:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1006,33 +1006,33 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 64b\n"
-      "cbz x26, 68f\n"
       "65:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 66f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "tbz x26, #0, 67f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
+      "cbz x27, 68f\n"
+      "tbz x27, #1, 66f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "tbz x27, #0, 67f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
       "b 67f\n"
       "66:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
       "67:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1040,31 +1040,31 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "68:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
-      "add v12.4s, v12.4s, v0.4s\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
       "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "add v10.4s, v10.4s, v2.4s\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "add v12.4s, v12.4s, v0.4s\n"
       "add v13.4s, v13.4s, v1.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add x14, x14, #0x40\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "tbz %x[flags], #4, 69f\n"
@@ -1075,20 +1075,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 70f\n"
       "69:"  // Height 3: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "70:"  // Height 3: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -1107,98 +1107,98 @@ void a64_hybrid_s8qs_dot_6x16 (
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
       "and v6.16b, v10.16b, v2.16b\n"
+      "and v7.16b, v11.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "and v7.16b, v11.16b, v3.16b\n"
+      "sqadd v11.4s, v11.4s, v7.4s\n"
       "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v12.4s, v12.4s, v4.4s\n"
-      "sqadd v13.4s, v13.4s, v5.4s\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
-      "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v12.4s, v12.4s, v4.4s\n"
+      "sqadd v13.4s, v13.4s, v5.4s\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "71:"  // Height 3: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v12.4s, v12.4s, v0.4s\n"
       "srshl v13.4s, v13.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v14.4s, v14.4s, v2.4s\n"
+      "srshl v15.4s, v15.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "srshl v14.4s, v14.4s, v2.4s\n"
-      "srshl v15.4s, v15.4s, v3.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
+      "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
       "smin v14.4s, v14.4s, v6.4s\n"
       "smin v15.4s, v15.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
       "smax v14.4s, v14.4s, v5.4s\n"
       "smax v15.4s, v15.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
-      "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "uzp1 v8.8h, v8.8h, v9.8h\n"
-      "add v18.4s, v18.4s, v4.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "uzp1 v9.8h, v10.8h, v11.8h\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "uzp1 v12.8h, v12.8h, v13.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v8.8h, v8.8h, v9.8h\n"
+      "uzp1 v9.8h, v10.8h, v11.8h\n"
+      "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
@@ -1207,96 +1207,96 @@ void a64_hybrid_s8qs_dot_6x16 (
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 80f\n"
       "tbz x10, #3, 75f\n"
-      "str d8, [x9], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
+      "str d8, [x11], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
       "tbz x10, #2, 73f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
       "tbz x10, #1, 72f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[14], [x9]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v8.b }[14], [x11]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
       "b 79f\n"
       "72:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[12], [x9]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v8.b }[12], [x11]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
       "b 79f\n"
       "73:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x10, #1, 74f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[10], [x9]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v8.b }[10], [x11]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
       "b 79f\n"
       "74:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[8], [x9]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v8.b }[8], [x11]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
       "b 79f\n"
       "75:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x10, #2, 77f\n"
-      "str s8, [x9], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
+      "str s8, [x11], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
       "tbz x10, #1, 76f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[6], [x9]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v8.b }[6], [x11]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
       "b 79f\n"
       "76:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[4], [x9]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v8.b }[4], [x11]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
       "b 79f\n"
       "77:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x10, #1, 78f\n"
-      "str h8, [x9], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
+      "str h8, [x11], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
       "tbz x10, #0, 79f\n"
-      "st1 { v8.b }[2], [x9]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v8.b }[2], [x11]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
       "b 79f\n"
       "78:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
       "79:"  // Height 3: Partial direct writeback: Done
       "b 81f\n"
       "80:"  // Height 3: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
       "81:"  // Height 3: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 56b\n"
       "b 164f\n"
       "82:"  // Height 4
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "83:"  // Height 4: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -1315,219 +1315,219 @@ void a64_hybrid_s8qs_dot_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "84:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "85:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 86f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 87f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 87f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 87f\n"
       "86:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "87:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 90f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 89f\n"
       "88:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "bge 88b\n"
       "89:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -1537,29 +1537,29 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       "90:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 95f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 95f\n"
+      "cmp x27, #0x4\n"
       "blt 92f\n"
       "91:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1569,38 +1569,38 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 91b\n"
-      "cbz x26, 95f\n"
       "92:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 93f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "tbz x26, #0, 94f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
+      "cbz x27, 95f\n"
+      "tbz x27, #1, 93f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "tbz x27, #0, 94f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
       "b 94f\n"
       "93:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
       "94:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1610,36 +1610,36 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "95:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 85b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
+      "add v9.4s, v9.4s, v1.4s\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
+      "add v10.4s, v10.4s, v2.4s\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x22, x23, x20\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
       "add v12.4s, v12.4s, v0.4s\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
-      "add v16.4s, v16.4s, v0.4s\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add v20.4s, v20.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
-      "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
-      "add v10.4s, v10.4s, v2.4s\n"
       "add v13.4s, v13.4s, v1.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
+      "add x14, x14, #0x40\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
       "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
@@ -1651,20 +1651,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 97f\n"
       "96:"  // Height 4: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "97:"  // Height 4: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -1687,125 +1687,125 @@ void a64_hybrid_s8qs_dot_6x16 (
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
       "and v6.16b, v10.16b, v2.16b\n"
+      "and v7.16b, v11.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "and v7.16b, v11.16b, v3.16b\n"
+      "sqadd v11.4s, v11.4s, v7.4s\n"
       "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v12.4s, v12.4s, v4.4s\n"
-      "sqadd v13.4s, v13.4s, v5.4s\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
-      "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v12.4s, v12.4s, v4.4s\n"
+      "sqadd v13.4s, v13.4s, v5.4s\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "and v4.16b, v20.16b, v0.16b\n"
       "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
+      "and v7.16b, v23.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
-      "and v7.16b, v23.16b, v3.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v7.4s\n"
       "98:"  // Height 4: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v12.4s, v12.4s, v0.4s\n"
       "srshl v13.4s, v13.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v14.4s, v14.4s, v2.4s\n"
+      "srshl v15.4s, v15.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
+      "srshl v20.4s, v20.4s, v0.4s\n"
+      "srshl v21.4s, v21.4s, v1.4s\n"
+      "srshl v22.4s, v22.4s, v2.4s\n"
+      "srshl v23.4s, v23.4s, v3.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "srshl v14.4s, v14.4s, v2.4s\n"
-      "srshl v15.4s, v15.4s, v3.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
+      "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "add v20.4s, v20.4s, v4.4s\n"
+      "add v21.4s, v21.4s, v4.4s\n"
+      "add v22.4s, v22.4s, v4.4s\n"
+      "add v23.4s, v23.4s, v4.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
       "smin v14.4s, v14.4s, v6.4s\n"
       "smin v15.4s, v15.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
       "smax v14.4s, v14.4s, v5.4s\n"
       "smax v15.4s, v15.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
-      "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "srshl v20.4s, v20.4s, v0.4s\n"
-      "add v18.4s, v18.4s, v4.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "add v20.4s, v20.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
-      "srshl v21.4s, v21.4s, v1.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v2.4s\n"
-      "srshl v23.4s, v23.4s, v3.4s\n"
-      "add v21.4s, v21.4s, v4.4s\n"
-      "uzp1 v8.8h, v8.8h, v9.8h\n"
-      "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "add v23.4s, v23.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
       "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
-      "uzp1 v9.8h, v10.8h, v11.8h\n"
       "smax v23.4s, v23.4s, v5.4s\n"
+      "uzp1 v8.8h, v8.8h, v9.8h\n"
+      "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
@@ -1818,112 +1818,112 @@ void a64_hybrid_s8qs_dot_6x16 (
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 107f\n"
       "tbz x10, #3, 102f\n"
-      "str d8, [x9], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
+      "str d8, [x11], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
       "tbz x10, #2, 100f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
       "tbz x10, #1, 99f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[14], [x9]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "st1 { v8.b }[14], [x11]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 106f\n"
       "99:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[12], [x9]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "st1 { v8.b }[12], [x11]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 106f\n"
       "100:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x10, #1, 101f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[10], [x9]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "st1 { v8.b }[10], [x11]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 106f\n"
       "101:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[8], [x9]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "st1 { v8.b }[8], [x11]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 106f\n"
       "102:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x10, #2, 104f\n"
-      "str s8, [x9], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
+      "str s8, [x11], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
       "tbz x10, #1, 103f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[6], [x9]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "st1 { v8.b }[6], [x11]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 106f\n"
       "103:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[4], [x9]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "st1 { v8.b }[4], [x11]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 106f\n"
       "104:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x10, #1, 105f\n"
-      "str h8, [x9], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
+      "str h8, [x11], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
       "tbz x10, #0, 106f\n"
-      "st1 { v8.b }[2], [x9]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "st1 { v8.b }[2], [x11]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 106f\n"
       "105:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "106:"  // Height 4: Partial direct writeback: Done
       "b 108f\n"
       "107:"  // Height 4: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
       "108:"  // Height 4: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 83b\n"
       "b 164f\n"
       "109:"  // Height 5
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "110:"  // Height 5: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -1946,258 +1946,258 @@ void a64_hybrid_s8qs_dot_6x16 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "111:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "112:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 113f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 114f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 114f\n"
       "113:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "114:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 117f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 116f\n"
       "115:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "bge 115b\n"
       "116:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2209,32 +2209,32 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       "117:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 122f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 122f\n"
+      "cmp x27, #0x4\n"
       "blt 119f\n"
       "118:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2246,43 +2246,43 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 118b\n"
-      "cbz x26, 122f\n"
       "119:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 120f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "tbz x26, #0, 121f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
+      "cbz x27, 122f\n"
+      "tbz x27, #1, 120f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "tbz x27, #0, 121f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
       "b 121f\n"
       "120:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
       "121:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2294,42 +2294,42 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "122:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 112b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
+      "add v9.4s, v9.4s, v1.4s\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
+      "add v10.4s, v10.4s, v2.4s\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
-      "add v12.4s, v12.4s, v0.4s\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
-      "add v16.4s, v16.4s, v0.4s\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "add v20.4s, v20.4s, v0.4s\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add v24.4s, v24.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
-      "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
-      "add v10.4s, v10.4s, v2.4s\n"
+      "add v12.4s, v12.4s, v0.4s\n"
       "add v13.4s, v13.4s, v1.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add x14, x14, #0x40\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
       "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
@@ -2341,20 +2341,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 124f\n"
       "123:"  // Height 5: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "124:"  // Height 5: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -2381,148 +2381,148 @@ void a64_hybrid_s8qs_dot_6x16 (
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
       "and v6.16b, v10.16b, v2.16b\n"
+      "and v7.16b, v11.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "and v7.16b, v11.16b, v3.16b\n"
+      "sqadd v11.4s, v11.4s, v7.4s\n"
       "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v12.4s, v12.4s, v4.4s\n"
-      "sqadd v13.4s, v13.4s, v5.4s\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
-      "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v12.4s, v12.4s, v4.4s\n"
+      "sqadd v13.4s, v13.4s, v5.4s\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "and v4.16b, v20.16b, v0.16b\n"
       "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
+      "and v7.16b, v23.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
-      "and v7.16b, v23.16b, v3.16b\n"
+      "sqadd v23.4s, v23.4s, v7.4s\n"
       "and v4.16b, v24.16b, v0.16b\n"
       "and v5.16b, v25.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v7.4s\n"
-      "sqadd v24.4s, v24.4s, v4.4s\n"
-      "sqadd v25.4s, v25.4s, v5.4s\n"
       "and v6.16b, v26.16b, v2.16b\n"
       "and v7.16b, v27.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v24.4s, v24.4s, v4.4s\n"
+      "sqadd v25.4s, v25.4s, v5.4s\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
       "125:"  // Height 5: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v12.4s, v12.4s, v0.4s\n"
       "srshl v13.4s, v13.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v14.4s, v14.4s, v2.4s\n"
+      "srshl v15.4s, v15.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
+      "srshl v20.4s, v20.4s, v0.4s\n"
+      "srshl v21.4s, v21.4s, v1.4s\n"
+      "srshl v22.4s, v22.4s, v2.4s\n"
+      "srshl v23.4s, v23.4s, v3.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v1.4s\n"
+      "srshl v26.4s, v26.4s, v2.4s\n"
+      "srshl v27.4s, v27.4s, v3.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "srshl v14.4s, v14.4s, v2.4s\n"
-      "srshl v15.4s, v15.4s, v3.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "srshl v20.4s, v20.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "srshl v21.4s, v21.4s, v1.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v2.4s\n"
-      "srshl v23.4s, v23.4s, v3.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "srshl v25.4s, v25.4s, v1.4s\n"
-      "smin v24.4s, v24.4s, v6.4s\n"
-      "srshl v26.4s, v26.4s, v2.4s\n"
-      "srshl v27.4s, v27.4s, v3.4s\n"
-      "smax v24.4s, v24.4s, v5.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
       "add v27.4s, v27.4s, v4.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
       "smin v26.4s, v26.4s, v6.4s\n"
       "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
@@ -2543,131 +2543,131 @@ void a64_hybrid_s8qs_dot_6x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 134f\n"
       "tbz x10, #3, 129f\n"
-      "str d8, [x9], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d8, [x11], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x10, #2, 127f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x10, #1, 126f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[14], [x9]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v8.b }[14], [x11]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 133f\n"
       "126:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[12], [x9]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v8.b }[12], [x11]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 133f\n"
       "127:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x10, #1, 128f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[10], [x9]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v8.b }[10], [x11]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 133f\n"
       "128:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[8], [x9]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v8.b }[8], [x11]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 133f\n"
       "129:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x10, #2, 131f\n"
-      "str s8, [x9], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s8, [x11], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x10, #1, 130f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[6], [x9]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v8.b }[6], [x11]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 133f\n"
       "130:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[4], [x9]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v8.b }[4], [x11]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 133f\n"
       "131:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x10, #1, 132f\n"
-      "str h8, [x9], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h8, [x11], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x10, #0, 133f\n"
-      "st1 { v8.b }[2], [x9]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v8.b }[2], [x11]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 133f\n"
       "132:"  // Height 5: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "133:"  // Height 5: Partial direct writeback: Done
       "b 135f\n"
       "134:"  // Height 5: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "135:"  // Height 5: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 110b\n"
       "b 164f\n"
       "136:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x6\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x20, #0x6\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "137:"  // Height 6: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -2694,297 +2694,297 @@ void a64_hybrid_s8qs_dot_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "138:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "139:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 140f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 141f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 141f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 141f\n"
       "140:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "141:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 144f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "blt 143f\n"
       "142:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8de  // sdot v30.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       "bge 142b\n"
       "143:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x9, #0x20]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x28, #0x70]\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x28, #0x80]\n"
+      "ldr q6, [x9, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x28, #0x90]\n"
+      "ldr q7, [x9, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x28, #0xa0]\n"
+      "ldr q6, [x9, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x28, #0xb0]\n"
+      "ldr q7, [x9, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x28, #0xc0]\n"
+      "ldr q6, [x9, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x28, #0xd0]\n"
+      "ldr q7, [x9, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x28, #0xe0]\n"
+      "ldr q6, [x9, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q7, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2998,35 +2998,35 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
       "144:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 149f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 149f\n"
+      "cmp x27, #0x4\n"
       "blt 146f\n"
       "145:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
-      "ldr q6, [x28, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3040,48 +3040,48 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 145b\n"
-      "cbz x26, 149f\n"
       "146:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 147f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "tbz x26, #0, 148f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "cbz x27, 149f\n"
+      "tbz x27, #1, 147f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "tbz x27, #0, 148f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 148f\n"
       "147:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "148:"  // Height 6: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x28, #0x0]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "ldr q7, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3095,48 +3095,48 @@ void a64_hybrid_s8qs_dot_6x16 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "149:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 139b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "add v8.4s, v8.4s, v0.4s\n"
+      "add v9.4s, v9.4s, v1.4s\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
+      "add v10.4s, v10.4s, v2.4s\n"
+      "add v11.4s, v11.4s, v3.4s\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
-      "add v12.4s, v12.4s, v0.4s\n"
+      "add x20, x21, x20\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
-      "add v16.4s, v16.4s, v0.4s\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "add v20.4s, v20.4s, v0.4s\n"
+      "add v12.4s, v12.4s, v0.4s\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "add v24.4s, v24.4s, v0.4s\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
-      "add v28.4s, v28.4s, v0.4s\n"
-      "ldr q1, [x11, #0x10]\n"
-      "ldr q2, [x11, #0x20]\n"
-      "add v9.4s, v9.4s, v1.4s\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
-      "add v10.4s, v10.4s, v2.4s\n"
       "add v13.4s, v13.4s, v1.4s\n"
       "add v14.4s, v14.4s, v2.4s\n"
-      "add v17.4s, v17.4s, v1.4s\n"
-      "add v11.4s, v11.4s, v3.4s\n"
+      "add x14, x14, #0x40\n"
       "add v15.4s, v15.4s, v3.4s\n"
+      "add v16.4s, v16.4s, v0.4s\n"
+      "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
       "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
+      "add v28.4s, v28.4s, v0.4s\n"
       "add v29.4s, v29.4s, v1.4s\n"
       "add v30.4s, v30.4s, v2.4s\n"
       "add v31.4s, v31.4s, v3.4s\n"
@@ -3148,20 +3148,20 @@ void a64_hybrid_s8qs_dot_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 151f\n"
       "150:"  // Height 6: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "151:"  // Height 6: parameters loaded
       "sqrdmulh v8.4s, v8.4s, v4.4s\n"
@@ -3192,183 +3192,183 @@ void a64_hybrid_s8qs_dot_6x16 (
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
       "and v6.16b, v10.16b, v2.16b\n"
+      "and v7.16b, v11.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v8.4s, v8.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "and v7.16b, v11.16b, v3.16b\n"
+      "sqadd v11.4s, v11.4s, v7.4s\n"
       "and v4.16b, v12.16b, v0.16b\n"
       "and v5.16b, v13.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v12.4s, v12.4s, v4.4s\n"
-      "sqadd v13.4s, v13.4s, v5.4s\n"
       "and v6.16b, v14.16b, v2.16b\n"
       "and v7.16b, v15.16b, v3.16b\n"
-      "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v12.4s, v12.4s, v4.4s\n"
+      "sqadd v13.4s, v13.4s, v5.4s\n"
       "sqadd v14.4s, v14.4s, v6.4s\n"
       "sqadd v15.4s, v15.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "and v4.16b, v20.16b, v0.16b\n"
       "and v5.16b, v21.16b, v1.16b\n"
       "and v6.16b, v22.16b, v2.16b\n"
+      "and v7.16b, v23.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v20.4s, v20.4s, v4.4s\n"
       "sqadd v21.4s, v21.4s, v5.4s\n"
       "sqadd v22.4s, v22.4s, v6.4s\n"
-      "and v7.16b, v23.16b, v3.16b\n"
+      "sqadd v23.4s, v23.4s, v7.4s\n"
       "and v4.16b, v24.16b, v0.16b\n"
       "and v5.16b, v25.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v7.4s\n"
-      "sqadd v24.4s, v24.4s, v4.4s\n"
-      "sqadd v25.4s, v25.4s, v5.4s\n"
       "and v6.16b, v26.16b, v2.16b\n"
       "and v7.16b, v27.16b, v3.16b\n"
-      "and v4.16b, v28.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v24.4s, v24.4s, v4.4s\n"
+      "sqadd v25.4s, v25.4s, v5.4s\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
-      "sqadd v28.4s, v28.4s, v4.4s\n"
+      "and v4.16b, v28.16b, v0.16b\n"
       "and v5.16b, v29.16b, v1.16b\n"
       "and v6.16b, v30.16b, v2.16b\n"
       "and v7.16b, v31.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v28.4s, v28.4s, v4.4s\n"
       "sqadd v29.4s, v29.4s, v5.4s\n"
       "sqadd v30.4s, v30.4s, v6.4s\n"
       "sqadd v31.4s, v31.4s, v7.4s\n"
       "152:"  // Height 6: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v12.4s, v12.4s, v0.4s\n"
       "srshl v13.4s, v13.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v14.4s, v14.4s, v2.4s\n"
+      "srshl v15.4s, v15.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
+      "srshl v20.4s, v20.4s, v0.4s\n"
+      "srshl v21.4s, v21.4s, v1.4s\n"
+      "srshl v22.4s, v22.4s, v2.4s\n"
+      "srshl v23.4s, v23.4s, v3.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v1.4s\n"
+      "srshl v26.4s, v26.4s, v2.4s\n"
+      "srshl v27.4s, v27.4s, v3.4s\n"
+      "srshl v28.4s, v28.4s, v0.4s\n"
+      "srshl v29.4s, v29.4s, v1.4s\n"
+      "srshl v30.4s, v30.4s, v2.4s\n"
+      "srshl v31.4s, v31.4s, v3.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
-      "srshl v14.4s, v14.4s, v2.4s\n"
-      "srshl v15.4s, v15.4s, v3.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "srshl v20.4s, v20.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "srshl v21.4s, v21.4s, v1.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v2.4s\n"
-      "srshl v23.4s, v23.4s, v3.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "srshl v25.4s, v25.4s, v1.4s\n"
-      "smin v24.4s, v24.4s, v6.4s\n"
-      "srshl v26.4s, v26.4s, v2.4s\n"
-      "srshl v27.4s, v27.4s, v3.4s\n"
-      "smax v24.4s, v24.4s, v5.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
       "add v27.4s, v27.4s, v4.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
-      "smax v25.4s, v25.4s, v5.4s\n"
-      "smax v26.4s, v26.4s, v5.4s\n"
-      "smax v27.4s, v27.4s, v5.4s\n"
-      "srshl v28.4s, v28.4s, v0.4s\n"
-      "srshl v29.4s, v29.4s, v1.4s\n"
-      "srshl v30.4s, v30.4s, v2.4s\n"
-      "srshl v31.4s, v31.4s, v3.4s\n"
       "add v28.4s, v28.4s, v4.4s\n"
       "add v29.4s, v29.4s, v4.4s\n"
       "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
       "smin v28.4s, v28.4s, v6.4s\n"
       "smin v29.4s, v29.4s, v6.4s\n"
       "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v24.4s, v24.4s, v5.4s\n"
+      "smax v25.4s, v25.4s, v5.4s\n"
+      "smax v26.4s, v26.4s, v5.4s\n"
+      "smax v27.4s, v27.4s, v5.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
       "uzp1 v12.8h, v12.8h, v13.8h\n"
       "uzp1 v13.8h, v14.8h, v15.8h\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
@@ -3385,152 +3385,152 @@ void a64_hybrid_s8qs_dot_6x16 (
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 161f\n"
       "tbz x10, #3, 156f\n"
-      "str d8, [x9], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
+      "str d8, [x11], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
       "tbz x10, #2, 154f\n"
-      "st1 { v8.s }[2], [x9], #0x4\n"
-      "st1 { v12.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
+      "st1 { v8.s }[2], [x11], #0x4\n"
+      "st1 { v12.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
       "tbz x10, #1, 153f\n"
-      "st1 { v8.h }[6], [x9], #0x2\n"
-      "st1 { v12.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
+      "st1 { v8.h }[6], [x11], #0x2\n"
+      "st1 { v12.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[14], [x9]\n"
-      "st1 { v12.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "st1 { v8.b }[14], [x11]\n"
+      "st1 { v12.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 160f\n"
       "153:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[12], [x9]\n"
-      "st1 { v12.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "st1 { v8.b }[12], [x11]\n"
+      "st1 { v12.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 160f\n"
       "154:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x10, #1, 155f\n"
-      "st1 { v8.h }[4], [x9], #0x2\n"
-      "st1 { v12.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
+      "st1 { v8.h }[4], [x11], #0x2\n"
+      "st1 { v12.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[10], [x9]\n"
-      "st1 { v12.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "st1 { v8.b }[10], [x11]\n"
+      "st1 { v12.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 160f\n"
       "155:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[8], [x9]\n"
-      "st1 { v12.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "st1 { v8.b }[8], [x11]\n"
+      "st1 { v12.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 160f\n"
       "156:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x10, #2, 158f\n"
-      "str s8, [x9], #0x4\n"
-      "str s12, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
+      "str s8, [x11], #0x4\n"
+      "str s12, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
       "tbz x10, #1, 157f\n"
-      "st1 { v8.h }[2], [x9], #0x2\n"
-      "st1 { v12.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
+      "st1 { v8.h }[2], [x11], #0x2\n"
+      "st1 { v12.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[6], [x9]\n"
-      "st1 { v12.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "st1 { v8.b }[6], [x11]\n"
+      "st1 { v12.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 160f\n"
       "157:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[4], [x9]\n"
-      "st1 { v12.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "st1 { v8.b }[4], [x11]\n"
+      "st1 { v12.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 160f\n"
       "158:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x10, #1, 159f\n"
-      "str h8, [x9], #0x2\n"
-      "str h12, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
+      "str h8, [x11], #0x2\n"
+      "str h12, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
       "tbz x10, #0, 160f\n"
-      "st1 { v8.b }[2], [x9]\n"
-      "st1 { v12.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "st1 { v8.b }[2], [x11]\n"
+      "st1 { v12.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 160f\n"
       "159:"  // Height 6: Partial direct writeback: partial_1_0
-      "str b8, [x9, #0x0]\n"
-      "str b12, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b8, [x11, #0x0]\n"
+      "str b12, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "160:"  // Height 6: Partial direct writeback: Done
       "b 162f\n"
       "161:"  // Height 6: Full writeback
-      "str q8, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q12, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q8, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q12, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "162:"  // Height 6: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 137b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 164f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 163f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "163:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "164:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp
index 8924492e41..fc525531b2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -95,12 +95,12 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 59f\n"
       "beq 30f\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[output_ptr]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -111,196 +111,192 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 9f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x9, #0x10]\n"
       "blt 8f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "add x9, x9, #0x100\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 7b\n"
       "8:"  // Height 1: Multiply loop: Single iteration only
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x28, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x9, x9, #0x100\n"
       "9:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 16f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 16f\n"
+      "cmp x27, #0x8\n"
       "blt 11f\n"
       "10:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr q6, [x9, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
+      "add x9, x9, #0x80\n"
       "bge 10b\n"
-      "cbz x26, 16f\n"
       "11:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 13f\n"
-      "ldr s1, [x25], #0x4\n"
-      "tbz x26, #1, 12f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "tbz x26, #0, 15f\n"
-      "ld1 { v1.b }[6], [x25]\n"
+      "cbz x27, 16f\n"
+      "tbz x27, #2, 13f\n"
+      "ldr s1, [x26], #0x4\n"
+      "tbz x27, #1, 12f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "tbz x27, #0, 15f\n"
+      "ld1 { v1.b }[6], [x26]\n"
       "b 15f\n"
       "12:"  // Height 1: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 15f\n"
-      "ld1 { v1.b }[4], [x25]\n"
+      "tbz x27, #0, 15f\n"
+      "ld1 { v1.b }[4], [x26]\n"
       "b 15f\n"
       "13:"  // Height 1: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 14f\n"
-      "ldr h1, [x25], #0x2\n"
-      "tbz x26, #0, 15f\n"
-      "ld1 { v1.b }[2], [x25]\n"
+      "tbz x27, #1, 14f\n"
+      "ldr h1, [x26], #0x2\n"
+      "tbz x27, #0, 15f\n"
+      "ld1 { v1.b }[2], [x26]\n"
       "b 15f\n"
       "14:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x28, #0x0]\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "add x9, x9, #0x80\n"
       "16:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 4b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
-      "ldr q1, [x11, #0x10]\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
-      "ldr q2, [x11, #0x20]\n"
       "mov v15.16b, v8.16b\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
       "add v15.4s, v15.4s, v0.4s\n"
+      "add x14, x14, #0x40\n"
       "add v9.4s, v9.4s, v1.4s\n"
       "add v10.4s, v10.4s, v2.4s\n"
       "add v11.4s, v11.4s, v3.4s\n"
@@ -312,20 +308,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 18f\n"
       "17:"  // Height 1: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "18:"  // Height 1: parameters loaded
       "sqrdmulh v15.4s, v15.4s, v4.4s\n"
@@ -340,98 +336,98 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v15.4s, v15.4s, v4.4s\n"
       "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
       "19:"  // Height 1: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v15.4s, v15.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v10.4s, v10.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v11.4s, v11.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
+      "cmp x10, #0x10\n"
       "smin v15.4s, v15.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
       "smax v15.4s, v15.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "uzp1 v15.8h, v15.8h, v9.8h\n"
       "smax v11.4s, v11.4s, v5.4s\n"
+      "uzp1 v15.8h, v15.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v15.16b, v15.16b, v9.16b\n"
       "bge 28f\n"
       "tbz x10, #3, 23f\n"
-      "str d15, [x9], #0x8\n"
+      "str d15, [x11], #0x8\n"
       "tbz x10, #2, 21f\n"
-      "st1 { v15.s }[2], [x9], #0x4\n"
+      "st1 { v15.s }[2], [x11], #0x4\n"
       "tbz x10, #1, 20f\n"
-      "st1 { v15.h }[6], [x9], #0x2\n"
+      "st1 { v15.h }[6], [x11], #0x2\n"
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[14], [x9]\n"
+      "st1 { v15.b }[14], [x11]\n"
       "b 27f\n"
       "20:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[12], [x9]\n"
+      "st1 { v15.b }[12], [x11]\n"
       "b 27f\n"
       "21:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x10, #1, 22f\n"
-      "st1 { v15.h }[4], [x9], #0x2\n"
+      "st1 { v15.h }[4], [x11], #0x2\n"
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[10], [x9]\n"
+      "st1 { v15.b }[10], [x11]\n"
       "b 27f\n"
       "22:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[8], [x9]\n"
+      "st1 { v15.b }[8], [x11]\n"
       "b 27f\n"
       "23:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x10, #2, 25f\n"
-      "str s15, [x9], #0x4\n"
+      "str s15, [x11], #0x4\n"
       "tbz x10, #1, 24f\n"
-      "st1 { v15.h }[2], [x9], #0x2\n"
+      "st1 { v15.h }[2], [x11], #0x2\n"
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[6], [x9]\n"
+      "st1 { v15.b }[6], [x11]\n"
       "b 27f\n"
       "24:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[4], [x9]\n"
+      "st1 { v15.b }[4], [x11]\n"
       "b 27f\n"
       "25:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x10, #1, 26f\n"
-      "str h15, [x9], #0x2\n"
+      "str h15, [x11], #0x2\n"
       "tbz x10, #0, 27f\n"
-      "st1 { v15.b }[2], [x9]\n"
+      "st1 { v15.b }[2], [x11]\n"
       "b 27f\n"
       "26:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b15, [x9, #0x0]\n"
+      "str b15, [x11, #0x0]\n"
       "27:"  // Height 1: Partial direct writeback: Done
       "b 29f\n"
       "28:"  // Height 1: Full writeback
-      "str q15, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
+      "str q15, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
       "29:"  // Height 1: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 2b\n"
       "b 176f\n"
       "30:"  // Height 2
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "31:"  // Height 2: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -442,216 +438,216 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "32:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "33:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 34f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 35f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 35f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 35f\n"
       "34:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "35:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 38f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "blt 37f\n"
       "36:"  // Height 2: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
+      "add x9, x9, #0x100\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "bge 36b\n"
       "37:"  // Height 2: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x9, x9, #0x100\n"
       "38:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 45f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 45f\n"
+      "cmp x27, #0x8\n"
       "blt 40f\n"
       "39:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x28, #0x10]\n"
-      "ldr q6, [x28, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x20]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x40]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x60]\n"
+      "ldr q7, [x9, #0x70]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
+      "add x9, x9, #0x80\n"
       "bge 39b\n"
-      "cbz x26, 45f\n"
       "40:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 42f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "tbz x26, #1, 41f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "tbz x26, #0, 44f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
+      "cbz x27, 45f\n"
+      "tbz x27, #2, 42f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "tbz x27, #1, 41f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "tbz x27, #0, 44f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
       "b 44f\n"
       "41:"  // Height 2: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 44f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
+      "tbz x27, #0, 44f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
       "b 44f\n"
       "42:"  // Height 2: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 43f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "tbz x26, #0, 44f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
+      "tbz x27, #1, 43f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "tbz x27, #0, 44f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
       "b 44f\n"
       "43:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
       "44:"  // Height 2: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "ldr q6, [x28, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "add x9, x9, #0x80\n"
       "45:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 33b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "ldr q1, [x11, #0x10]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "ldr q2, [x11, #0x20]\n"
+      "add x24, x11, x20\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "mov v15.16b, v7.16b\n"
       "add v15.4s, v15.4s, v0.4s\n"
+      "add x14, x14, #0x40\n"
       "add v12.4s, v12.4s, v1.4s\n"
       "add v13.4s, v13.4s, v2.4s\n"
       "add v14.4s, v14.4s, v3.4s\n"
@@ -667,20 +663,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 47f\n"
       "46:"  // Height 2: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "47:"  // Height 2: parameters loaded
       "sqrdmulh v15.4s, v15.4s, v4.4s\n"
@@ -695,149 +691,149 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "and v4.16b, v15.16b, v0.16b\n"
       "and v5.16b, v12.16b, v1.16b\n"
       "and v6.16b, v13.16b, v2.16b\n"
+      "and v7.16b, v14.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v15.4s, v15.4s, v4.4s\n"
       "sqadd v12.4s, v12.4s, v5.4s\n"
       "sqadd v13.4s, v13.4s, v6.4s\n"
-      "and v7.16b, v14.16b, v3.16b\n"
+      "sqadd v14.4s, v14.4s, v7.4s\n"
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v14.4s, v14.4s, v7.4s\n"
-      "sqadd v8.4s, v8.4s, v4.4s\n"
-      "sqadd v9.4s, v9.4s, v5.4s\n"
       "and v6.16b, v10.16b, v2.16b\n"
       "and v7.16b, v11.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v8.4s, v8.4s, v4.4s\n"
+      "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
       "48:"  // Height 2: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v15.4s, v15.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v12.4s, v12.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v13.4s, v13.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v14.4s, v14.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v10.4s, v10.4s, v2.4s\n"
+      "srshl v11.4s, v11.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
       "add v15.4s, v15.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
+      "add v10.4s, v10.4s, v4.4s\n"
+      "add v11.4s, v11.4s, v4.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
       "smin v14.4s, v14.4s, v6.4s\n"
       "smin v8.4s, v8.4s, v6.4s\n"
       "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
       "smax v14.4s, v14.4s, v5.4s\n"
       "smax v8.4s, v8.4s, v5.4s\n"
       "smax v9.4s, v9.4s, v5.4s\n"
-      "srshl v10.4s, v10.4s, v2.4s\n"
-      "srshl v11.4s, v11.4s, v3.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
       "uzp1 v15.8h, v15.8h, v12.8h\n"
       "uzp1 v12.8h, v13.8h, v14.8h\n"
-      "add v10.4s, v10.4s, v4.4s\n"
-      "add v11.4s, v11.4s, v4.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "uzp1 v15.16b, v15.16b, v12.16b\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
+      "uzp1 v15.16b, v15.16b, v12.16b\n"
       "uzp1 v8.16b, v8.16b, v9.16b\n"
       "bge 57f\n"
       "tbz x10, #3, 52f\n"
-      "str d15, [x9], #0x8\n"
-      "str d8, [x23], #0x8\n"
+      "str d15, [x11], #0x8\n"
+      "str d8, [x24], #0x8\n"
       "tbz x10, #2, 50f\n"
-      "st1 { v15.s }[2], [x9], #0x4\n"
-      "st1 { v8.s }[2], [x23], #0x4\n"
+      "st1 { v15.s }[2], [x11], #0x4\n"
+      "st1 { v8.s }[2], [x24], #0x4\n"
       "tbz x10, #1, 49f\n"
-      "st1 { v15.h }[6], [x9], #0x2\n"
-      "st1 { v8.h }[6], [x23], #0x2\n"
+      "st1 { v15.h }[6], [x11], #0x2\n"
+      "st1 { v8.h }[6], [x24], #0x2\n"
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[14], [x9]\n"
-      "st1 { v8.b }[14], [x23]\n"
+      "st1 { v15.b }[14], [x11]\n"
+      "st1 { v8.b }[14], [x24]\n"
       "b 56f\n"
       "49:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[12], [x9]\n"
-      "st1 { v8.b }[12], [x23]\n"
+      "st1 { v15.b }[12], [x11]\n"
+      "st1 { v8.b }[12], [x24]\n"
       "b 56f\n"
       "50:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x10, #1, 51f\n"
-      "st1 { v15.h }[4], [x9], #0x2\n"
-      "st1 { v8.h }[4], [x23], #0x2\n"
+      "st1 { v15.h }[4], [x11], #0x2\n"
+      "st1 { v8.h }[4], [x24], #0x2\n"
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[10], [x9]\n"
-      "st1 { v8.b }[10], [x23]\n"
+      "st1 { v15.b }[10], [x11]\n"
+      "st1 { v8.b }[10], [x24]\n"
       "b 56f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[8], [x9]\n"
-      "st1 { v8.b }[8], [x23]\n"
+      "st1 { v15.b }[8], [x11]\n"
+      "st1 { v8.b }[8], [x24]\n"
       "b 56f\n"
       "52:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x10, #2, 54f\n"
-      "str s15, [x9], #0x4\n"
-      "str s8, [x23], #0x4\n"
+      "str s15, [x11], #0x4\n"
+      "str s8, [x24], #0x4\n"
       "tbz x10, #1, 53f\n"
-      "st1 { v15.h }[2], [x9], #0x2\n"
-      "st1 { v8.h }[2], [x23], #0x2\n"
+      "st1 { v15.h }[2], [x11], #0x2\n"
+      "st1 { v8.h }[2], [x24], #0x2\n"
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[6], [x9]\n"
-      "st1 { v8.b }[6], [x23]\n"
+      "st1 { v15.b }[6], [x11]\n"
+      "st1 { v8.b }[6], [x24]\n"
       "b 56f\n"
       "53:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[4], [x9]\n"
-      "st1 { v8.b }[4], [x23]\n"
+      "st1 { v15.b }[4], [x11]\n"
+      "st1 { v8.b }[4], [x24]\n"
       "b 56f\n"
       "54:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x10, #1, 55f\n"
-      "str h15, [x9], #0x2\n"
-      "str h8, [x23], #0x2\n"
+      "str h15, [x11], #0x2\n"
+      "str h8, [x24], #0x2\n"
       "tbz x10, #0, 56f\n"
-      "st1 { v15.b }[2], [x9]\n"
-      "st1 { v8.b }[2], [x23]\n"
+      "st1 { v15.b }[2], [x11]\n"
+      "st1 { v8.b }[2], [x24]\n"
       "b 56f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b15, [x9, #0x0]\n"
-      "str b8, [x23, #0x0]\n"
+      "str b15, [x11, #0x0]\n"
+      "str b8, [x24, #0x0]\n"
       "56:"  // Height 2: Partial direct writeback: Done
       "b 58f\n"
       "57:"  // Height 2: Full writeback
-      "str q15, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q8, [x23, #0x0]\n"
+      "str q15, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q8, [x24, #0x0]\n"
       "58:"  // Height 2: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 31b\n"
       "b 176f\n"
       "59:"  // Height 3
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "60:"  // Height 3: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -856,295 +852,291 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "61:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "62:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 63f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 64f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 64f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 64f\n"
       "63:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "64:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 67f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "blt 66f\n"
       "65:"  // Height 3: Multiply loop: Main loop head
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "bge 65b\n"
       "66:"  // Height 3: Multiply loop: Single iteration only
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       "67:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 74f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 74f\n"
+      "cmp x27, #0x8\n"
       "blt 69f\n"
       "68:"  // Height 3: Multiply loop: Odd block loop
-      "movi v4.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr q6, [x9, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       "bge 68b\n"
-      "cbz x26, 74f\n"
       "69:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 71f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "tbz x26, #1, 70f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "tbz x26, #0, 73f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
+      "cbz x27, 74f\n"
+      "tbz x27, #2, 71f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "tbz x27, #1, 70f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "tbz x27, #0, 73f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
       "b 73f\n"
       "70:"  // Height 3: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 73f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
+      "tbz x27, #0, 73f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
       "b 73f\n"
       "71:"  // Height 3: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 72f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "tbz x26, #0, 73f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
+      "tbz x27, #1, 72f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "tbz x27, #0, 73f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
       "b 73f\n"
       "72:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
       "73:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v4.16b, #0x0\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x28, #0x10]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       "74:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 62b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "ldr q1, [x11, #0x10]\n"
+      "add x24, x11, x20\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "ldr q2, [x11, #0x20]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "add x23, x24, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
       "uzp1 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
+      "add x14, x14, #0x40\n"
       "mov v23.16b, v7.16b\n"
       "add v23.4s, v23.4s, v0.4s\n"
       "add v12.4s, v12.4s, v1.4s\n"
@@ -1166,20 +1158,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 76f\n"
       "75:"  // Height 3: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "76:"  // Height 3: parameters loaded
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
@@ -1198,98 +1190,98 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v12.16b, v1.16b\n"
       "and v6.16b, v13.16b, v2.16b\n"
+      "and v7.16b, v14.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v12.4s, v12.4s, v5.4s\n"
       "sqadd v13.4s, v13.4s, v6.4s\n"
-      "and v7.16b, v14.16b, v3.16b\n"
+      "sqadd v14.4s, v14.4s, v7.4s\n"
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v14.4s, v14.4s, v7.4s\n"
-      "sqadd v8.4s, v8.4s, v4.4s\n"
-      "sqadd v9.4s, v9.4s, v5.4s\n"
       "and v6.16b, v10.16b, v2.16b\n"
       "and v7.16b, v11.16b, v3.16b\n"
-      "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v8.4s, v8.4s, v4.4s\n"
+      "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
       "and v7.16b, v19.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "77:"  // Height 3: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v12.4s, v12.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v13.4s, v13.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v14.4s, v14.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v10.4s, v10.4s, v2.4s\n"
+      "srshl v11.4s, v11.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "srshl v10.4s, v10.4s, v2.4s\n"
-      "srshl v11.4s, v11.4s, v3.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
+      "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
       "smin v11.4s, v11.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
       "smax v11.4s, v11.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
-      "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "uzp1 v23.8h, v23.8h, v12.8h\n"
-      "add v18.4s, v18.4s, v4.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "uzp1 v12.8h, v13.8h, v14.8h\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "uzp1 v8.8h, v8.8h, v9.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v23.8h, v23.8h, v12.8h\n"
+      "uzp1 v12.8h, v13.8h, v14.8h\n"
+      "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
@@ -1298,96 +1290,96 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 86f\n"
       "tbz x10, #3, 81f\n"
-      "str d23, [x9], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
+      "str d23, [x11], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
       "tbz x10, #2, 79f\n"
-      "st1 { v23.s }[2], [x9], #0x4\n"
-      "st1 { v8.s }[2], [x23], #0x4\n"
-      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v23.s }[2], [x11], #0x4\n"
+      "st1 { v8.s }[2], [x24], #0x4\n"
+      "st1 { v16.s }[2], [x23], #0x4\n"
       "tbz x10, #1, 78f\n"
-      "st1 { v23.h }[6], [x9], #0x2\n"
-      "st1 { v8.h }[6], [x23], #0x2\n"
-      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v23.h }[6], [x11], #0x2\n"
+      "st1 { v8.h }[6], [x24], #0x2\n"
+      "st1 { v16.h }[6], [x23], #0x2\n"
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[14], [x9]\n"
-      "st1 { v8.b }[14], [x23]\n"
-      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v23.b }[14], [x11]\n"
+      "st1 { v8.b }[14], [x24]\n"
+      "st1 { v16.b }[14], [x23]\n"
       "b 85f\n"
       "78:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[12], [x9]\n"
-      "st1 { v8.b }[12], [x23]\n"
-      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v23.b }[12], [x11]\n"
+      "st1 { v8.b }[12], [x24]\n"
+      "st1 { v16.b }[12], [x23]\n"
       "b 85f\n"
       "79:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x10, #1, 80f\n"
-      "st1 { v23.h }[4], [x9], #0x2\n"
-      "st1 { v8.h }[4], [x23], #0x2\n"
-      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v23.h }[4], [x11], #0x2\n"
+      "st1 { v8.h }[4], [x24], #0x2\n"
+      "st1 { v16.h }[4], [x23], #0x2\n"
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[10], [x9]\n"
-      "st1 { v8.b }[10], [x23]\n"
-      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v23.b }[10], [x11]\n"
+      "st1 { v8.b }[10], [x24]\n"
+      "st1 { v16.b }[10], [x23]\n"
       "b 85f\n"
       "80:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[8], [x9]\n"
-      "st1 { v8.b }[8], [x23]\n"
-      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v23.b }[8], [x11]\n"
+      "st1 { v8.b }[8], [x24]\n"
+      "st1 { v16.b }[8], [x23]\n"
       "b 85f\n"
       "81:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x10, #2, 83f\n"
-      "str s23, [x9], #0x4\n"
-      "str s8, [x23], #0x4\n"
-      "str s16, [x22], #0x4\n"
+      "str s23, [x11], #0x4\n"
+      "str s8, [x24], #0x4\n"
+      "str s16, [x23], #0x4\n"
       "tbz x10, #1, 82f\n"
-      "st1 { v23.h }[2], [x9], #0x2\n"
-      "st1 { v8.h }[2], [x23], #0x2\n"
-      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v23.h }[2], [x11], #0x2\n"
+      "st1 { v8.h }[2], [x24], #0x2\n"
+      "st1 { v16.h }[2], [x23], #0x2\n"
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[6], [x9]\n"
-      "st1 { v8.b }[6], [x23]\n"
-      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v23.b }[6], [x11]\n"
+      "st1 { v8.b }[6], [x24]\n"
+      "st1 { v16.b }[6], [x23]\n"
       "b 85f\n"
       "82:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[4], [x9]\n"
-      "st1 { v8.b }[4], [x23]\n"
-      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v23.b }[4], [x11]\n"
+      "st1 { v8.b }[4], [x24]\n"
+      "st1 { v16.b }[4], [x23]\n"
       "b 85f\n"
       "83:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x10, #1, 84f\n"
-      "str h23, [x9], #0x2\n"
-      "str h8, [x23], #0x2\n"
-      "str h16, [x22], #0x2\n"
+      "str h23, [x11], #0x2\n"
+      "str h8, [x24], #0x2\n"
+      "str h16, [x23], #0x2\n"
       "tbz x10, #0, 85f\n"
-      "st1 { v23.b }[2], [x9]\n"
-      "st1 { v8.b }[2], [x23]\n"
-      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v23.b }[2], [x11]\n"
+      "st1 { v8.b }[2], [x24]\n"
+      "st1 { v16.b }[2], [x23]\n"
       "b 85f\n"
       "84:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b23, [x9, #0x0]\n"
-      "str b8, [x23, #0x0]\n"
-      "str b16, [x22, #0x0]\n"
+      "str b23, [x11, #0x0]\n"
+      "str b8, [x24, #0x0]\n"
+      "str b16, [x23, #0x0]\n"
       "85:"  // Height 3: Partial direct writeback: Done
       "b 87f\n"
       "86:"  // Height 3: Full writeback
-      "str q23, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q8, [x23, #0x0]\n"
-      "str q16, [x22, #0x0]\n"
+      "str q23, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q8, [x24, #0x0]\n"
+      "str q16, [x23, #0x0]\n"
       "87:"  // Height 3: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 60b\n"
       "b 176f\n"
       "88:"  // Height 4
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "89:"  // Height 4: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -1406,312 +1398,312 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "90:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "91:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 92f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 93f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 93f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 93f\n"
       "92:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "93:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 96f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "blt 95f\n"
       "94:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x9, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "bge 94b\n"
       "95:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       "96:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 103f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 103f\n"
+      "cmp x27, #0x8\n"
       "blt 98f\n"
       "97:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "cmp x27, #0x8\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       "bge 97b\n"
-      "cbz x26, 103f\n"
       "98:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 100f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "tbz x26, #1, 99f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "tbz x26, #0, 102f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
+      "cbz x27, 103f\n"
+      "tbz x27, #2, 100f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "tbz x27, #1, 99f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "tbz x27, #0, 102f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
       "b 102f\n"
       "99:"  // Height 4: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 102f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
+      "tbz x27, #0, 102f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
       "b 102f\n"
       "100:"  // Height 4: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 101f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "tbz x26, #0, 102f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "tbz x27, #1, 101f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "tbz x27, #0, 102f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 102f\n"
       "101:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "102:"  // Height 4: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x9, #0x0]\n"
+      "ldr q6, [x9, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       "103:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 91b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "add x24, x11, x20\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "ldr q1, [x11, #0x10]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "ldr q2, [x11, #0x20]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
+      "add x14, x14, #0x40\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "mov v23.16b, v7.16b\n"
@@ -1739,20 +1731,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 105f\n"
       "104:"  // Height 4: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "105:"  // Height 4: parameters loaded
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
@@ -1775,125 +1767,125 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v12.16b, v1.16b\n"
       "and v6.16b, v13.16b, v2.16b\n"
+      "and v7.16b, v14.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v12.4s, v12.4s, v5.4s\n"
       "sqadd v13.4s, v13.4s, v6.4s\n"
-      "and v7.16b, v14.16b, v3.16b\n"
+      "sqadd v14.4s, v14.4s, v7.4s\n"
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v14.4s, v14.4s, v7.4s\n"
-      "sqadd v8.4s, v8.4s, v4.4s\n"
-      "sqadd v9.4s, v9.4s, v5.4s\n"
       "and v6.16b, v10.16b, v2.16b\n"
       "and v7.16b, v11.16b, v3.16b\n"
-      "and v4.16b, v15.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v8.4s, v8.4s, v4.4s\n"
+      "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v15.4s, v15.4s, v4.4s\n"
+      "and v4.16b, v15.16b, v0.16b\n"
       "and v5.16b, v20.16b, v1.16b\n"
       "and v6.16b, v21.16b, v2.16b\n"
       "and v7.16b, v22.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v15.4s, v15.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
       "sqadd v21.4s, v21.4s, v6.4s\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
+      "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "and v7.16b, v19.16b, v3.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "106:"  // Height 4: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v12.4s, v12.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v13.4s, v13.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v14.4s, v14.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v10.4s, v10.4s, v2.4s\n"
+      "srshl v11.4s, v11.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v15.4s, v15.4s, v0.4s\n"
+      "srshl v20.4s, v20.4s, v1.4s\n"
+      "srshl v21.4s, v21.4s, v2.4s\n"
+      "srshl v22.4s, v22.4s, v3.4s\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "srshl v10.4s, v10.4s, v2.4s\n"
-      "srshl v11.4s, v11.4s, v3.4s\n"
-      "srshl v15.4s, v15.4s, v0.4s\n"
-      "srshl v20.4s, v20.4s, v1.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
+      "add v20.4s, v20.4s, v4.4s\n"
+      "add v21.4s, v21.4s, v4.4s\n"
+      "add v22.4s, v22.4s, v4.4s\n"
+      "add v16.4s, v16.4s, v4.4s\n"
+      "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
       "smin v10.4s, v10.4s, v6.4s\n"
       "smin v11.4s, v11.4s, v6.4s\n"
       "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
       "smax v10.4s, v10.4s, v5.4s\n"
       "smax v11.4s, v11.4s, v5.4s\n"
       "smax v15.4s, v15.4s, v5.4s\n"
-      "add v20.4s, v20.4s, v4.4s\n"
-      "srshl v21.4s, v21.4s, v2.4s\n"
-      "srshl v22.4s, v22.4s, v3.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
-      "add v21.4s, v21.4s, v4.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
-      "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "add v16.4s, v16.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
       "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
-      "add v17.4s, v17.4s, v4.4s\n"
-      "uzp1 v23.8h, v23.8h, v12.8h\n"
-      "add v18.4s, v18.4s, v4.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "uzp1 v12.8h, v13.8h, v14.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v23.8h, v23.8h, v12.8h\n"
+      "uzp1 v12.8h, v13.8h, v14.8h\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
       "uzp1 v15.8h, v15.8h, v20.8h\n"
@@ -1906,112 +1898,112 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 115f\n"
       "tbz x10, #3, 110f\n"
-      "str d23, [x9], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
+      "str d23, [x11], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
       "tbz x10, #2, 108f\n"
-      "st1 { v23.s }[2], [x9], #0x4\n"
-      "st1 { v8.s }[2], [x23], #0x4\n"
-      "st1 { v15.s }[2], [x22], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
+      "st1 { v23.s }[2], [x11], #0x4\n"
+      "st1 { v8.s }[2], [x24], #0x4\n"
+      "st1 { v15.s }[2], [x23], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
       "tbz x10, #1, 107f\n"
-      "st1 { v23.h }[6], [x9], #0x2\n"
-      "st1 { v8.h }[6], [x23], #0x2\n"
-      "st1 { v15.h }[6], [x22], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
+      "st1 { v23.h }[6], [x11], #0x2\n"
+      "st1 { v8.h }[6], [x24], #0x2\n"
+      "st1 { v15.h }[6], [x23], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[14], [x9]\n"
-      "st1 { v8.b }[14], [x23]\n"
-      "st1 { v15.b }[14], [x22]\n"
-      "st1 { v16.b }[14], [x21]\n"
+      "st1 { v23.b }[14], [x11]\n"
+      "st1 { v8.b }[14], [x24]\n"
+      "st1 { v15.b }[14], [x23]\n"
+      "st1 { v16.b }[14], [x22]\n"
       "b 114f\n"
       "107:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[12], [x9]\n"
-      "st1 { v8.b }[12], [x23]\n"
-      "st1 { v15.b }[12], [x22]\n"
-      "st1 { v16.b }[12], [x21]\n"
+      "st1 { v23.b }[12], [x11]\n"
+      "st1 { v8.b }[12], [x24]\n"
+      "st1 { v15.b }[12], [x23]\n"
+      "st1 { v16.b }[12], [x22]\n"
       "b 114f\n"
       "108:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x10, #1, 109f\n"
-      "st1 { v23.h }[4], [x9], #0x2\n"
-      "st1 { v8.h }[4], [x23], #0x2\n"
-      "st1 { v15.h }[4], [x22], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
+      "st1 { v23.h }[4], [x11], #0x2\n"
+      "st1 { v8.h }[4], [x24], #0x2\n"
+      "st1 { v15.h }[4], [x23], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[10], [x9]\n"
-      "st1 { v8.b }[10], [x23]\n"
-      "st1 { v15.b }[10], [x22]\n"
-      "st1 { v16.b }[10], [x21]\n"
+      "st1 { v23.b }[10], [x11]\n"
+      "st1 { v8.b }[10], [x24]\n"
+      "st1 { v15.b }[10], [x23]\n"
+      "st1 { v16.b }[10], [x22]\n"
       "b 114f\n"
       "109:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[8], [x9]\n"
-      "st1 { v8.b }[8], [x23]\n"
-      "st1 { v15.b }[8], [x22]\n"
-      "st1 { v16.b }[8], [x21]\n"
+      "st1 { v23.b }[8], [x11]\n"
+      "st1 { v8.b }[8], [x24]\n"
+      "st1 { v15.b }[8], [x23]\n"
+      "st1 { v16.b }[8], [x22]\n"
       "b 114f\n"
       "110:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x10, #2, 112f\n"
-      "str s23, [x9], #0x4\n"
-      "str s8, [x23], #0x4\n"
-      "str s15, [x22], #0x4\n"
-      "str s16, [x21], #0x4\n"
+      "str s23, [x11], #0x4\n"
+      "str s8, [x24], #0x4\n"
+      "str s15, [x23], #0x4\n"
+      "str s16, [x22], #0x4\n"
       "tbz x10, #1, 111f\n"
-      "st1 { v23.h }[2], [x9], #0x2\n"
-      "st1 { v8.h }[2], [x23], #0x2\n"
-      "st1 { v15.h }[2], [x22], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
+      "st1 { v23.h }[2], [x11], #0x2\n"
+      "st1 { v8.h }[2], [x24], #0x2\n"
+      "st1 { v15.h }[2], [x23], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[6], [x9]\n"
-      "st1 { v8.b }[6], [x23]\n"
-      "st1 { v15.b }[6], [x22]\n"
-      "st1 { v16.b }[6], [x21]\n"
+      "st1 { v23.b }[6], [x11]\n"
+      "st1 { v8.b }[6], [x24]\n"
+      "st1 { v15.b }[6], [x23]\n"
+      "st1 { v16.b }[6], [x22]\n"
       "b 114f\n"
       "111:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[4], [x9]\n"
-      "st1 { v8.b }[4], [x23]\n"
-      "st1 { v15.b }[4], [x22]\n"
-      "st1 { v16.b }[4], [x21]\n"
+      "st1 { v23.b }[4], [x11]\n"
+      "st1 { v8.b }[4], [x24]\n"
+      "st1 { v15.b }[4], [x23]\n"
+      "st1 { v16.b }[4], [x22]\n"
       "b 114f\n"
       "112:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x10, #1, 113f\n"
-      "str h23, [x9], #0x2\n"
-      "str h8, [x23], #0x2\n"
-      "str h15, [x22], #0x2\n"
-      "str h16, [x21], #0x2\n"
+      "str h23, [x11], #0x2\n"
+      "str h8, [x24], #0x2\n"
+      "str h15, [x23], #0x2\n"
+      "str h16, [x22], #0x2\n"
       "tbz x10, #0, 114f\n"
-      "st1 { v23.b }[2], [x9]\n"
-      "st1 { v8.b }[2], [x23]\n"
-      "st1 { v15.b }[2], [x22]\n"
-      "st1 { v16.b }[2], [x21]\n"
+      "st1 { v23.b }[2], [x11]\n"
+      "st1 { v8.b }[2], [x24]\n"
+      "st1 { v15.b }[2], [x23]\n"
+      "st1 { v16.b }[2], [x22]\n"
       "b 114f\n"
       "113:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b23, [x9, #0x0]\n"
-      "str b8, [x23, #0x0]\n"
-      "str b15, [x22, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
+      "str b23, [x11, #0x0]\n"
+      "str b8, [x24, #0x0]\n"
+      "str b15, [x23, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
       "114:"  // Height 4: Partial direct writeback: Done
       "b 116f\n"
       "115:"  // Height 4: Full writeback
-      "str q23, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q8, [x23, #0x0]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q16, [x21, #0x0]\n"
+      "str q23, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q8, [x24, #0x0]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q16, [x22, #0x0]\n"
       "116:"  // Height 4: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 89b\n"
       "b 176f\n"
       "117:"  // Height 5
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "118:"  // Height 5: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -2038,212 +2030,210 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "119:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "120:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 122f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 122f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 122f\n"
       "121:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "122:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 125f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "blt 124f\n"
       "123:"  // Height 5: Multiply loop: Main loop head
-      "movi v6.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "cmp x26, #0x20\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
       "bge 123b\n"
       "124:"  // Height 5: Multiply loop: Single iteration only
-      "movi v6.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
@@ -2251,136 +2241,134 @@ void a64_hybrid_s8qs_mmla_6x16 (
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
       "125:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 132f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 132f\n"
+      "cmp x27, #0x8\n"
       "blt 127f\n"
       "126:"  // Height 5: Multiply loop: Odd block loop
-      "movi v7.4s, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr q6, [x28, #0x0]\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr q6, [x9, #0x0]\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q7, [x28, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a498  // smmla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e87a49c  // smmla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a499  // smmla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49d  // smmla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49a  // smmla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49e  // smmla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49b  // smmla v27.4s, v4.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49f  // smmla v31.4s, v4.16b, v7.16b\n"
       "bge 126b\n"
-      "cbz x26, 132f\n"
       "127:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 129f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "tbz x26, #1, 128f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "tbz x26, #0, 131f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
+      "cbz x27, 132f\n"
+      "tbz x27, #2, 129f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "tbz x27, #1, 128f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "tbz x27, #0, 131f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
       "b 131f\n"
       "128:"  // Height 5: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 131f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
+      "tbz x27, #0, 131f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
       "b 131f\n"
       "129:"  // Height 5: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 130f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "tbz x26, #0, 131f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
+      "tbz x27, #1, 130f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "tbz x27, #0, 131f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
       "b 131f\n"
       "130:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
       "131:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "movi v6.4s, #0x0\n"
-      "ldr q7, [x28, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
@@ -2388,41 +2376,41 @@ void a64_hybrid_s8qs_mmla_6x16 (
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
       "132:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 120b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "ldr q1, [x11, #0x10]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
-      "ldr q2, [x11, #0x20]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
+      "add x14, x14, #0x40\n"
       "uzp1 v24.2d, v24.2d, v28.2d\n"
       "uzp1 v25.2d, v25.2d, v29.2d\n"
       "uzp1 v26.2d, v26.2d, v30.2d\n"
@@ -2456,20 +2444,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 134f\n"
       "133:"  // Height 5: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "134:"  // Height 5: parameters loaded
       "sqrdmulh v31.4s, v31.4s, v4.4s\n"
@@ -2496,148 +2484,148 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v12.16b, v1.16b\n"
       "and v6.16b, v13.16b, v2.16b\n"
+      "and v7.16b, v14.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v12.4s, v12.4s, v5.4s\n"
       "sqadd v13.4s, v13.4s, v6.4s\n"
-      "and v7.16b, v14.16b, v3.16b\n"
+      "sqadd v14.4s, v14.4s, v7.4s\n"
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v14.4s, v14.4s, v7.4s\n"
-      "sqadd v8.4s, v8.4s, v4.4s\n"
-      "sqadd v9.4s, v9.4s, v5.4s\n"
       "and v6.16b, v10.16b, v2.16b\n"
       "and v7.16b, v11.16b, v3.16b\n"
-      "and v4.16b, v15.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v8.4s, v8.4s, v4.4s\n"
+      "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v15.4s, v15.4s, v4.4s\n"
+      "and v4.16b, v15.16b, v0.16b\n"
       "and v5.16b, v20.16b, v1.16b\n"
       "and v6.16b, v21.16b, v2.16b\n"
       "and v7.16b, v22.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v15.4s, v15.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
       "sqadd v21.4s, v21.4s, v6.4s\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
+      "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "and v7.16b, v19.16b, v3.16b\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
       "and v4.16b, v24.16b, v0.16b\n"
       "and v5.16b, v25.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sqadd v24.4s, v24.4s, v4.4s\n"
-      "sqadd v25.4s, v25.4s, v5.4s\n"
       "and v6.16b, v26.16b, v2.16b\n"
       "and v7.16b, v27.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v24.4s, v24.4s, v4.4s\n"
+      "sqadd v25.4s, v25.4s, v5.4s\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
       "135:"  // Height 5: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v12.4s, v12.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v13.4s, v13.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v14.4s, v14.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v10.4s, v10.4s, v2.4s\n"
+      "srshl v11.4s, v11.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v15.4s, v15.4s, v0.4s\n"
+      "srshl v20.4s, v20.4s, v1.4s\n"
+      "srshl v21.4s, v21.4s, v2.4s\n"
+      "srshl v22.4s, v22.4s, v3.4s\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v1.4s\n"
+      "srshl v26.4s, v26.4s, v2.4s\n"
+      "srshl v27.4s, v27.4s, v3.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "srshl v10.4s, v10.4s, v2.4s\n"
-      "srshl v11.4s, v11.4s, v3.4s\n"
-      "srshl v15.4s, v15.4s, v0.4s\n"
-      "srshl v20.4s, v20.4s, v1.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
-      "srshl v21.4s, v21.4s, v2.4s\n"
-      "srshl v22.4s, v22.4s, v3.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "srshl v25.4s, v25.4s, v1.4s\n"
-      "smin v24.4s, v24.4s, v6.4s\n"
-      "srshl v26.4s, v26.4s, v2.4s\n"
-      "srshl v27.4s, v27.4s, v3.4s\n"
-      "smax v24.4s, v24.4s, v5.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
       "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
       "smin v26.4s, v26.4s, v6.4s\n"
       "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
@@ -2658,131 +2646,131 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 144f\n"
       "tbz x10, #3, 139f\n"
-      "str d31, [x9], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d31, [x11], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x10, #2, 137f\n"
-      "st1 { v31.s }[2], [x9], #0x4\n"
-      "st1 { v8.s }[2], [x23], #0x4\n"
-      "st1 { v15.s }[2], [x22], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v31.s }[2], [x11], #0x4\n"
+      "st1 { v8.s }[2], [x24], #0x4\n"
+      "st1 { v15.s }[2], [x23], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x10, #1, 136f\n"
-      "st1 { v31.h }[6], [x9], #0x2\n"
-      "st1 { v8.h }[6], [x23], #0x2\n"
-      "st1 { v15.h }[6], [x22], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v31.h }[6], [x11], #0x2\n"
+      "st1 { v8.h }[6], [x24], #0x2\n"
+      "st1 { v15.h }[6], [x23], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[14], [x9]\n"
-      "st1 { v8.b }[14], [x23]\n"
-      "st1 { v15.b }[14], [x22]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v31.b }[14], [x11]\n"
+      "st1 { v8.b }[14], [x24]\n"
+      "st1 { v15.b }[14], [x23]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 143f\n"
       "136:"  // Height 5: Partial direct writeback: partial_1_12
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[12], [x9]\n"
-      "st1 { v8.b }[12], [x23]\n"
-      "st1 { v15.b }[12], [x22]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v31.b }[12], [x11]\n"
+      "st1 { v8.b }[12], [x24]\n"
+      "st1 { v15.b }[12], [x23]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 143f\n"
       "137:"  // Height 5: Partial direct writeback: partial_2_8
       "tbz x10, #1, 138f\n"
-      "st1 { v31.h }[4], [x9], #0x2\n"
-      "st1 { v8.h }[4], [x23], #0x2\n"
-      "st1 { v15.h }[4], [x22], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v31.h }[4], [x11], #0x2\n"
+      "st1 { v8.h }[4], [x24], #0x2\n"
+      "st1 { v15.h }[4], [x23], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[10], [x9]\n"
-      "st1 { v8.b }[10], [x23]\n"
-      "st1 { v15.b }[10], [x22]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v31.b }[10], [x11]\n"
+      "st1 { v8.b }[10], [x24]\n"
+      "st1 { v15.b }[10], [x23]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 143f\n"
       "138:"  // Height 5: Partial direct writeback: partial_1_8
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[8], [x9]\n"
-      "st1 { v8.b }[8], [x23]\n"
-      "st1 { v15.b }[8], [x22]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v31.b }[8], [x11]\n"
+      "st1 { v8.b }[8], [x24]\n"
+      "st1 { v15.b }[8], [x23]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 143f\n"
       "139:"  // Height 5: Partial direct writeback: partial_4_0
       "tbz x10, #2, 141f\n"
-      "str s31, [x9], #0x4\n"
-      "str s8, [x23], #0x4\n"
-      "str s15, [x22], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s31, [x11], #0x4\n"
+      "str s8, [x24], #0x4\n"
+      "str s15, [x23], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x10, #1, 140f\n"
-      "st1 { v31.h }[2], [x9], #0x2\n"
-      "st1 { v8.h }[2], [x23], #0x2\n"
-      "st1 { v15.h }[2], [x22], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v31.h }[2], [x11], #0x2\n"
+      "st1 { v8.h }[2], [x24], #0x2\n"
+      "st1 { v15.h }[2], [x23], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[6], [x9]\n"
-      "st1 { v8.b }[6], [x23]\n"
-      "st1 { v15.b }[6], [x22]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v31.b }[6], [x11]\n"
+      "st1 { v8.b }[6], [x24]\n"
+      "st1 { v15.b }[6], [x23]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 143f\n"
       "140:"  // Height 5: Partial direct writeback: partial_1_4
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[4], [x9]\n"
-      "st1 { v8.b }[4], [x23]\n"
-      "st1 { v15.b }[4], [x22]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v31.b }[4], [x11]\n"
+      "st1 { v8.b }[4], [x24]\n"
+      "st1 { v15.b }[4], [x23]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 143f\n"
       "141:"  // Height 5: Partial direct writeback: partial_2_0
       "tbz x10, #1, 142f\n"
-      "str h31, [x9], #0x2\n"
-      "str h8, [x23], #0x2\n"
-      "str h15, [x22], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h31, [x11], #0x2\n"
+      "str h8, [x24], #0x2\n"
+      "str h15, [x23], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x10, #0, 143f\n"
-      "st1 { v31.b }[2], [x9]\n"
-      "st1 { v8.b }[2], [x23]\n"
-      "st1 { v15.b }[2], [x22]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v31.b }[2], [x11]\n"
+      "st1 { v8.b }[2], [x24]\n"
+      "st1 { v15.b }[2], [x23]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 143f\n"
       "142:"  // Height 5: Partial direct writeback: partial_1_0
-      "str b31, [x9, #0x0]\n"
-      "str b8, [x23, #0x0]\n"
-      "str b15, [x22, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b31, [x11, #0x0]\n"
+      "str b8, [x24, #0x0]\n"
+      "str b15, [x23, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "143:"  // Height 5: Partial direct writeback: Done
       "b 145f\n"
       "144:"  // Height 5: Full writeback
-      "str q31, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q8, [x23, #0x0]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q31, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q8, [x24, #0x0]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "145:"  // Height 5: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 118b\n"
       "b 176f\n"
       "146:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x6\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x20, #0x6\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "147:"  // Height 6: Column loop
       "movi v8.4s, #0x0\n"
       "movi v9.4s, #0x0\n"
@@ -2809,219 +2797,219 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "148:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "149:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 150f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 151f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 151f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 151f\n"
       "150:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "151:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 154f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr q7, [x9, #0x0]\n"
       "blt 153f\n"
       "152:"  // Height 6: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x9, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 152b\n"
       "153:"  // Height 6: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x80]\n"
+      "ldr q7, [x9, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x90]\n"
+      "ldr q6, [x9, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xa0]\n"
+      "ldr q7, [x9, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xb0]\n"
+      "ldr q6, [x9, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xc0]\n"
+      "ldr q7, [x9, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xd0]\n"
+      "ldr q6, [x9, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x28, #0xe0]\n"
+      "ldr q7, [x9, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "ldr q6, [x9, #0xf0]\n"
+      "add x9, x9, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
@@ -3029,48 +3017,48 @@ void a64_hybrid_s8qs_mmla_6x16 (
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
       "154:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 161f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 161f\n"
+      "cmp x27, #0x8\n"
       "blt 156f\n"
       "155:"  // Height 6: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "cmp x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q6, [x28, #0x0]\n"
-      "ldr q7, [x28, #0x10]\n"
+      "ldr q6, [x9, #0x0]\n"
+      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a498  // smmla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x20]\n"
+      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49c  // smmla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x30]\n"
+      "ldr q7, [x9, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a499  // smmla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x40]\n"
+      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49d  // smmla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x50]\n"
+      "ldr q7, [x9, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49a  // smmla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x60]\n"
+      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49e  // smmla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q7, [x9, #0x70]\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49b  // smmla v27.4s, v4.16b, v6.16b\n"
@@ -3078,136 +3066,136 @@ void a64_hybrid_s8qs_mmla_6x16 (
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49f  // smmla v31.4s, v4.16b, v7.16b\n"
       "bge 155b\n"
-      "cbz x26, 161f\n"
       "156:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 158f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "ldr s6, [x20], #0x4\n"
-      "tbz x26, #1, 157f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "ld1 { v6.h }[2], [x20], #0x2\n"
-      "tbz x26, #0, 160f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
-      "ld1 { v6.b }[6], [x20]\n"
+      "cbz x27, 161f\n"
+      "tbz x27, #2, 158f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "ldr s6, [x21], #0x4\n"
+      "tbz x27, #1, 157f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "ld1 { v6.h }[2], [x21], #0x2\n"
+      "tbz x27, #0, 160f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
+      "ld1 { v6.b }[6], [x21]\n"
       "b 160f\n"
       "157:"  // Height 6: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 160f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
-      "ld1 { v6.b }[4], [x20]\n"
+      "tbz x27, #0, 160f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
+      "ld1 { v6.b }[4], [x21]\n"
       "b 160f\n"
       "158:"  // Height 6: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 159f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "ldr h6, [x20], #0x2\n"
-      "tbz x26, #0, 160f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
-      "ld1 { v6.b }[2], [x20]\n"
+      "tbz x27, #1, 159f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "ldr h6, [x21], #0x2\n"
+      "tbz x27, #0, 160f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
+      "ld1 { v6.b }[2], [x21]\n"
       "b 160f\n"
       "159:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
-      "ldr b6, [x20, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
+      "ldr b6, [x21, #0x0]\n"
       "160:"  // Height 6: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x9, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x28, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x28, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "trn1 v4.2d, v5.2d, v6.2d\n"
+      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q7, [x9, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x30]\n"
+      "ldr q6, [x9, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x40]\n"
+      "ldr q7, [x9, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x50]\n"
+      "ldr q6, [x9, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x28, #0x60]\n"
+      "ldr q7, [x9, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
+      "ldr q6, [x9, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x9, x9, #0x80\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
       "161:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 149b\n"
+      "ldr q0, [x14, #0x0]\n"
+      "ldr q1, [x14, #0x10]\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x9, #0x0]\n"
-      "add x23, x9, x19\n"
+      "ldr q2, [x14, #0x20]\n"
+      "ldr q3, [x14, #0x30]\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "ldr q0, [x11, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
-      "ldr q1, [x11, #0x10]\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x11, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "ldr q2, [x11, #0x20]\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "ldr q3, [x11, #0x30]\n"
-      "add x11, x11, #0x40\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
+      "prfm pstl1keep, [x20, #0x0]\n"
+      "add x14, x14, #0x40\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "uzp1 v23.2d, v24.2d, v28.2d\n"
       "uzp2 v24.2d, v24.2d, v28.2d\n"
@@ -3250,20 +3238,20 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "ldr q2, [x12, #0x20]\n"
       "ldr q6, [x13, #0x20]\n"
       "ldr q3, [x12, #0x30]\n"
-      "add x12, x12, #0x40\n"
       "ldr q7, [x13, #0x30]\n"
+      "add x12, x12, #0x40\n"
       "add x13, x13, #0x40\n"
       "b 163f\n"
       "162:"  // Height 6: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1r { v0.4s }, [x24]\n"
+      "add x25, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x25]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "mov v1.16b, v0.16b\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1r { v4.4s }, [x24]\n"
-      "mov v2.16b, v0.16b\n"
-      "mov v3.16b, v0.16b\n"
       "mov v5.16b, v4.16b\n"
+      "mov v2.16b, v0.16b\n"
       "mov v6.16b, v4.16b\n"
+      "mov v3.16b, v0.16b\n"
       "mov v7.16b, v4.16b\n"
       "163:"  // Height 6: parameters loaded
       "sqrdmulh v31.4s, v31.4s, v4.4s\n"
@@ -3294,183 +3282,183 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v12.16b, v1.16b\n"
       "and v6.16b, v13.16b, v2.16b\n"
+      "and v7.16b, v14.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v12.4s, v12.4s, v5.4s\n"
       "sqadd v13.4s, v13.4s, v6.4s\n"
-      "and v7.16b, v14.16b, v3.16b\n"
+      "sqadd v14.4s, v14.4s, v7.4s\n"
       "and v4.16b, v8.16b, v0.16b\n"
       "and v5.16b, v9.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v14.4s, v14.4s, v7.4s\n"
-      "sqadd v8.4s, v8.4s, v4.4s\n"
-      "sqadd v9.4s, v9.4s, v5.4s\n"
       "and v6.16b, v10.16b, v2.16b\n"
       "and v7.16b, v11.16b, v3.16b\n"
-      "and v4.16b, v15.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v8.4s, v8.4s, v4.4s\n"
+      "sqadd v9.4s, v9.4s, v5.4s\n"
       "sqadd v10.4s, v10.4s, v6.4s\n"
       "sqadd v11.4s, v11.4s, v7.4s\n"
-      "sqadd v15.4s, v15.4s, v4.4s\n"
+      "and v4.16b, v15.16b, v0.16b\n"
       "and v5.16b, v20.16b, v1.16b\n"
       "and v6.16b, v21.16b, v2.16b\n"
       "and v7.16b, v22.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v15.4s, v15.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
       "sqadd v21.4s, v21.4s, v6.4s\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v1.16b\n"
       "and v6.16b, v18.16b, v2.16b\n"
+      "and v7.16b, v19.16b, v3.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "and v7.16b, v19.16b, v3.16b\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v28.16b, v1.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
-      "sqadd v28.4s, v28.4s, v5.4s\n"
       "and v6.16b, v29.16b, v2.16b\n"
       "and v7.16b, v30.16b, v3.16b\n"
-      "and v4.16b, v24.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
+      "sqadd v28.4s, v28.4s, v5.4s\n"
       "sqadd v29.4s, v29.4s, v6.4s\n"
       "sqadd v30.4s, v30.4s, v7.4s\n"
-      "sqadd v24.4s, v24.4s, v4.4s\n"
+      "and v4.16b, v24.16b, v0.16b\n"
       "and v5.16b, v25.16b, v1.16b\n"
       "and v6.16b, v26.16b, v2.16b\n"
       "and v7.16b, v27.16b, v3.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
+      "sqadd v24.4s, v24.4s, v4.4s\n"
       "sqadd v25.4s, v25.4s, v5.4s\n"
       "sqadd v26.4s, v26.4s, v6.4s\n"
       "sqadd v27.4s, v27.4s, v7.4s\n"
       "164:"  // Height 6: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x25]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x24]\n"
       "srshl v12.4s, v12.4s, v1.4s\n"
-      "add x24, %x[qp], %[minval]\n"
       "srshl v13.4s, v13.4s, v2.4s\n"
-      "ld1r { v5.4s }, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
       "srshl v14.4s, v14.4s, v3.4s\n"
-      "ld1r { v6.4s }, [x24]\n"
-      "cmp x10, #0x10\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x25]\n"
       "srshl v8.4s, v8.4s, v0.4s\n"
       "srshl v9.4s, v9.4s, v1.4s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x25]\n"
+      "srshl v10.4s, v10.4s, v2.4s\n"
+      "srshl v11.4s, v11.4s, v3.4s\n"
+      "cmp x10, #0x10\n"
+      "srshl v15.4s, v15.4s, v0.4s\n"
+      "srshl v20.4s, v20.4s, v1.4s\n"
+      "srshl v21.4s, v21.4s, v2.4s\n"
+      "srshl v22.4s, v22.4s, v3.4s\n"
+      "srshl v16.4s, v16.4s, v0.4s\n"
+      "srshl v17.4s, v17.4s, v1.4s\n"
+      "srshl v18.4s, v18.4s, v2.4s\n"
+      "srshl v19.4s, v19.4s, v3.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "srshl v28.4s, v28.4s, v1.4s\n"
+      "srshl v29.4s, v29.4s, v2.4s\n"
+      "srshl v30.4s, v30.4s, v3.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v1.4s\n"
+      "srshl v26.4s, v26.4s, v2.4s\n"
+      "srshl v27.4s, v27.4s, v3.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v12.4s, v12.4s, v4.4s\n"
       "add v13.4s, v13.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v12.4s, v12.4s, v6.4s\n"
-      "smin v13.4s, v13.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v12.4s, v12.4s, v5.4s\n"
-      "smax v13.4s, v13.4s, v5.4s\n"
       "add v14.4s, v14.4s, v4.4s\n"
       "add v8.4s, v8.4s, v4.4s\n"
       "add v9.4s, v9.4s, v4.4s\n"
-      "smin v14.4s, v14.4s, v6.4s\n"
-      "smin v8.4s, v8.4s, v6.4s\n"
-      "smin v9.4s, v9.4s, v6.4s\n"
-      "smax v14.4s, v14.4s, v5.4s\n"
-      "smax v8.4s, v8.4s, v5.4s\n"
-      "smax v9.4s, v9.4s, v5.4s\n"
-      "srshl v10.4s, v10.4s, v2.4s\n"
-      "srshl v11.4s, v11.4s, v3.4s\n"
-      "srshl v15.4s, v15.4s, v0.4s\n"
-      "srshl v20.4s, v20.4s, v1.4s\n"
       "add v10.4s, v10.4s, v4.4s\n"
       "add v11.4s, v11.4s, v4.4s\n"
       "add v15.4s, v15.4s, v4.4s\n"
-      "smin v10.4s, v10.4s, v6.4s\n"
-      "smin v11.4s, v11.4s, v6.4s\n"
-      "smin v15.4s, v15.4s, v6.4s\n"
-      "smax v10.4s, v10.4s, v5.4s\n"
-      "smax v11.4s, v11.4s, v5.4s\n"
-      "smax v15.4s, v15.4s, v5.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
-      "srshl v21.4s, v21.4s, v2.4s\n"
-      "srshl v22.4s, v22.4s, v3.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "srshl v16.4s, v16.4s, v0.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "srshl v17.4s, v17.4s, v1.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v2.4s\n"
-      "srshl v19.4s, v19.4s, v3.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "srshl v28.4s, v28.4s, v1.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "srshl v29.4s, v29.4s, v2.4s\n"
-      "srshl v30.4s, v30.4s, v3.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
       "add v28.4s, v28.4s, v4.4s\n"
       "add v29.4s, v29.4s, v4.4s\n"
       "add v30.4s, v30.4s, v4.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
-      "smax v28.4s, v28.4s, v5.4s\n"
-      "smax v29.4s, v29.4s, v5.4s\n"
-      "smax v30.4s, v30.4s, v5.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v1.4s\n"
-      "srshl v26.4s, v26.4s, v2.4s\n"
-      "srshl v27.4s, v27.4s, v3.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
       "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v12.4s, v12.4s, v6.4s\n"
+      "smin v13.4s, v13.4s, v6.4s\n"
+      "smin v14.4s, v14.4s, v6.4s\n"
+      "smin v8.4s, v8.4s, v6.4s\n"
+      "smin v9.4s, v9.4s, v6.4s\n"
+      "smin v10.4s, v10.4s, v6.4s\n"
+      "smin v11.4s, v11.4s, v6.4s\n"
+      "smin v15.4s, v15.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
       "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v12.4s, v12.4s, v5.4s\n"
+      "smax v13.4s, v13.4s, v5.4s\n"
+      "smax v14.4s, v14.4s, v5.4s\n"
+      "smax v8.4s, v8.4s, v5.4s\n"
+      "smax v9.4s, v9.4s, v5.4s\n"
+      "smax v10.4s, v10.4s, v5.4s\n"
+      "smax v11.4s, v11.4s, v5.4s\n"
+      "smax v15.4s, v15.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v28.4s, v28.4s, v5.4s\n"
+      "smax v29.4s, v29.4s, v5.4s\n"
+      "smax v30.4s, v30.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
+      "smax v27.4s, v27.4s, v5.4s\n"
       "uzp1 v31.8h, v31.8h, v12.8h\n"
       "uzp1 v12.8h, v13.8h, v14.8h\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "uzp1 v8.8h, v8.8h, v9.8h\n"
       "uzp1 v9.8h, v10.8h, v11.8h\n"
-      "smax v27.4s, v27.4s, v5.4s\n"
       "uzp1 v15.8h, v15.8h, v20.8h\n"
       "uzp1 v20.8h, v21.8h, v22.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
@@ -3487,152 +3475,152 @@ void a64_hybrid_s8qs_mmla_6x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 173f\n"
       "tbz x10, #3, 168f\n"
-      "str d31, [x9], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "str d24, [x19], #0x8\n"
+      "str d31, [x11], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "str d24, [x20], #0x8\n"
       "tbz x10, #2, 166f\n"
-      "st1 { v31.s }[2], [x9], #0x4\n"
-      "st1 { v8.s }[2], [x23], #0x4\n"
-      "st1 { v15.s }[2], [x22], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v23.s }[2], [x20], #0x4\n"
-      "st1 { v24.s }[2], [x19], #0x4\n"
+      "st1 { v31.s }[2], [x11], #0x4\n"
+      "st1 { v8.s }[2], [x24], #0x4\n"
+      "st1 { v15.s }[2], [x23], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v23.s }[2], [x21], #0x4\n"
+      "st1 { v24.s }[2], [x20], #0x4\n"
       "tbz x10, #1, 165f\n"
-      "st1 { v31.h }[6], [x9], #0x2\n"
-      "st1 { v8.h }[6], [x23], #0x2\n"
-      "st1 { v15.h }[6], [x22], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v23.h }[6], [x20], #0x2\n"
-      "st1 { v24.h }[6], [x19], #0x2\n"
+      "st1 { v31.h }[6], [x11], #0x2\n"
+      "st1 { v8.h }[6], [x24], #0x2\n"
+      "st1 { v15.h }[6], [x23], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v23.h }[6], [x21], #0x2\n"
+      "st1 { v24.h }[6], [x20], #0x2\n"
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[14], [x9]\n"
-      "st1 { v8.b }[14], [x23]\n"
-      "st1 { v15.b }[14], [x22]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v23.b }[14], [x20]\n"
-      "st1 { v24.b }[14], [x19]\n"
+      "st1 { v31.b }[14], [x11]\n"
+      "st1 { v8.b }[14], [x24]\n"
+      "st1 { v15.b }[14], [x23]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v23.b }[14], [x21]\n"
+      "st1 { v24.b }[14], [x20]\n"
       "b 172f\n"
       "165:"  // Height 6: Partial direct writeback: partial_1_12
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[12], [x9]\n"
-      "st1 { v8.b }[12], [x23]\n"
-      "st1 { v15.b }[12], [x22]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v23.b }[12], [x20]\n"
-      "st1 { v24.b }[12], [x19]\n"
+      "st1 { v31.b }[12], [x11]\n"
+      "st1 { v8.b }[12], [x24]\n"
+      "st1 { v15.b }[12], [x23]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v23.b }[12], [x21]\n"
+      "st1 { v24.b }[12], [x20]\n"
       "b 172f\n"
       "166:"  // Height 6: Partial direct writeback: partial_2_8
       "tbz x10, #1, 167f\n"
-      "st1 { v31.h }[4], [x9], #0x2\n"
-      "st1 { v8.h }[4], [x23], #0x2\n"
-      "st1 { v15.h }[4], [x22], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v23.h }[4], [x20], #0x2\n"
-      "st1 { v24.h }[4], [x19], #0x2\n"
+      "st1 { v31.h }[4], [x11], #0x2\n"
+      "st1 { v8.h }[4], [x24], #0x2\n"
+      "st1 { v15.h }[4], [x23], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v23.h }[4], [x21], #0x2\n"
+      "st1 { v24.h }[4], [x20], #0x2\n"
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[10], [x9]\n"
-      "st1 { v8.b }[10], [x23]\n"
-      "st1 { v15.b }[10], [x22]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v23.b }[10], [x20]\n"
-      "st1 { v24.b }[10], [x19]\n"
+      "st1 { v31.b }[10], [x11]\n"
+      "st1 { v8.b }[10], [x24]\n"
+      "st1 { v15.b }[10], [x23]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v23.b }[10], [x21]\n"
+      "st1 { v24.b }[10], [x20]\n"
       "b 172f\n"
       "167:"  // Height 6: Partial direct writeback: partial_1_8
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[8], [x9]\n"
-      "st1 { v8.b }[8], [x23]\n"
-      "st1 { v15.b }[8], [x22]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v23.b }[8], [x20]\n"
-      "st1 { v24.b }[8], [x19]\n"
+      "st1 { v31.b }[8], [x11]\n"
+      "st1 { v8.b }[8], [x24]\n"
+      "st1 { v15.b }[8], [x23]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v23.b }[8], [x21]\n"
+      "st1 { v24.b }[8], [x20]\n"
       "b 172f\n"
       "168:"  // Height 6: Partial direct writeback: partial_4_0
       "tbz x10, #2, 170f\n"
-      "str s31, [x9], #0x4\n"
-      "str s8, [x23], #0x4\n"
-      "str s15, [x22], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s23, [x20], #0x4\n"
-      "str s24, [x19], #0x4\n"
+      "str s31, [x11], #0x4\n"
+      "str s8, [x24], #0x4\n"
+      "str s15, [x23], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s23, [x21], #0x4\n"
+      "str s24, [x20], #0x4\n"
       "tbz x10, #1, 169f\n"
-      "st1 { v31.h }[2], [x9], #0x2\n"
-      "st1 { v8.h }[2], [x23], #0x2\n"
-      "st1 { v15.h }[2], [x22], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v23.h }[2], [x20], #0x2\n"
-      "st1 { v24.h }[2], [x19], #0x2\n"
+      "st1 { v31.h }[2], [x11], #0x2\n"
+      "st1 { v8.h }[2], [x24], #0x2\n"
+      "st1 { v15.h }[2], [x23], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v23.h }[2], [x21], #0x2\n"
+      "st1 { v24.h }[2], [x20], #0x2\n"
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[6], [x9]\n"
-      "st1 { v8.b }[6], [x23]\n"
-      "st1 { v15.b }[6], [x22]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v23.b }[6], [x20]\n"
-      "st1 { v24.b }[6], [x19]\n"
+      "st1 { v31.b }[6], [x11]\n"
+      "st1 { v8.b }[6], [x24]\n"
+      "st1 { v15.b }[6], [x23]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v23.b }[6], [x21]\n"
+      "st1 { v24.b }[6], [x20]\n"
       "b 172f\n"
       "169:"  // Height 6: Partial direct writeback: partial_1_4
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[4], [x9]\n"
-      "st1 { v8.b }[4], [x23]\n"
-      "st1 { v15.b }[4], [x22]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v23.b }[4], [x20]\n"
-      "st1 { v24.b }[4], [x19]\n"
+      "st1 { v31.b }[4], [x11]\n"
+      "st1 { v8.b }[4], [x24]\n"
+      "st1 { v15.b }[4], [x23]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v23.b }[4], [x21]\n"
+      "st1 { v24.b }[4], [x20]\n"
       "b 172f\n"
       "170:"  // Height 6: Partial direct writeback: partial_2_0
       "tbz x10, #1, 171f\n"
-      "str h31, [x9], #0x2\n"
-      "str h8, [x23], #0x2\n"
-      "str h15, [x22], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h23, [x20], #0x2\n"
-      "str h24, [x19], #0x2\n"
+      "str h31, [x11], #0x2\n"
+      "str h8, [x24], #0x2\n"
+      "str h15, [x23], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h23, [x21], #0x2\n"
+      "str h24, [x20], #0x2\n"
       "tbz x10, #0, 172f\n"
-      "st1 { v31.b }[2], [x9]\n"
-      "st1 { v8.b }[2], [x23]\n"
-      "st1 { v15.b }[2], [x22]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v23.b }[2], [x20]\n"
-      "st1 { v24.b }[2], [x19]\n"
+      "st1 { v31.b }[2], [x11]\n"
+      "st1 { v8.b }[2], [x24]\n"
+      "st1 { v15.b }[2], [x23]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v23.b }[2], [x21]\n"
+      "st1 { v24.b }[2], [x20]\n"
       "b 172f\n"
       "171:"  // Height 6: Partial direct writeback: partial_1_0
-      "str b31, [x9, #0x0]\n"
-      "str b8, [x23, #0x0]\n"
-      "str b15, [x22, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b23, [x20, #0x0]\n"
-      "str b24, [x19, #0x0]\n"
+      "str b31, [x11, #0x0]\n"
+      "str b8, [x24, #0x0]\n"
+      "str b15, [x23, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b23, [x21, #0x0]\n"
+      "str b24, [x20, #0x0]\n"
       "172:"  // Height 6: Partial direct writeback: Done
       "b 174f\n"
       "173:"  // Height 6: Full writeback
-      "str q31, [x9, #0x0]\n"
-      "add x9, x9, #0x10\n"
-      "str q8, [x23, #0x0]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q23, [x20, #0x0]\n"
-      "str q24, [x19, #0x0]\n"
+      "str q31, [x11, #0x0]\n"
+      "add x11, x11, #0x10\n"
+      "str q8, [x24, #0x0]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q23, [x21, #0x0]\n"
+      "str q24, [x20, #0x0]\n"
       "174:"  // Height 6: Writeback done
       "subs x10, x10, #0x10\n"
       "bgt 147b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 176f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 175f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "175:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "176:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp
index 3817785a79..8046b2ebb0 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 69f\n"
       "beq 35f\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 12f\n"
-      "cmp x17, #0x10\n"
+      "cmp x8, #0x10\n"
       "bge 11f\n"
-      "tbz x17, #3, 6f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "tbz x17, #2, 4f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "tbz x8, #3, 6f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "tbz x8, #2, 4f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 3f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v11.s }[2], [x17]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s11, [x17, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x17, #1, 5f\n"
-      "ldr d10, [x15], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "tbz x8, #1, 5f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v10.s }[2], [x17]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s10, [x17, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x17, #2, 8f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 7f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "tbz x8, #2, 8f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 7f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v9.s }[2], [x17]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s9, [x17, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x17, #1, 9f\n"
-      "ldr d8, [x15], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "tbz x8, #1, 9f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v8.s }[2], [x17]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 13f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
       "b 13f\n"
       "12:"  // Height 1: no accumulate
       "movi v8.4s, #0x0\n"
@@ -161,112 +161,109 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
       "13:"  // Height 1: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "14:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 15f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "cbnz x14, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "cbnz x15, 16f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
       "b 16f\n"
       "15:"  // Height 1: setup direct input
-      "mov x12, %x[input_ptr]\n"
+      "mov x13, %x[input_ptr]\n"
       "16:"  // Height 1: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 19f\n"
-      "ldr q0, [x12, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
       "ldr q6, [x16, #0x0]\n"
-      "cmp x13, #0x20\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 18f\n"
       "17:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
-      "ldr x11, [x16, #0x18]\n"
-      "add x12, x12, #0x10\n"
       "ldr d6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "ldr x10, [x16, #0x28]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
-      "ldr x10, [x16, #0x48]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
-      "ldr x10, [x16, #0x68]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x78]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
-      "ldr x10, [x16, #0x88]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x98]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
-      "ldr x10, [x16, #0xa8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xb8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
-      "ldr x10, [x16, #0xc8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
-      "ldr x10, [x16, #0xe8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xf0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xf8]\n"
+      "mov v7.d[1], x11\n"
+      "add x13, x13, #0x10\n"
       "add x16, x16, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
-      "ldr x10, [x16, #0x8]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d0, [x12, #0x0]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d0, [x13, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "bge 17b\n"
       "18:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
       "ldr q6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
+      ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
@@ -290,200 +287,203 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "ldr q6, [x16, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       "ldr q7, [x16, #0xf0]\n"
+      "add x13, x13, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "add x16, x16, #0x100\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
+      "add x16, x16, #0x100\n"
       "19:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x13, 24f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 24f\n"
+      "cmp x14, #0x4\n"
       "blt 21f\n"
       "20:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "cmp x13, #0x4\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x16, #0x20]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "ldr q6, [x16, #0x20]\n"
+      "cmp x14, #0x4\n"
       "ldr q7, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x16, x16, #0x40\n"
       "bge 20b\n"
-      "cbz x13, 24f\n"
       "21:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 22f\n"
-      "ldr h0, [x12], #0x2\n"
-      "tbz x13, #0, 23f\n"
-      "ld1 { v0.b }[2], [x12]\n"
+      "cbz x14, 24f\n"
+      "tbz x14, #1, 22f\n"
+      "ldr h0, [x13], #0x2\n"
+      "tbz x14, #0, 23f\n"
+      "ld1 { v0.b }[2], [x13]\n"
       "b 23f\n"
       "22:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x16, #0x20]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
+      "ldr q6, [x16, #0x20]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x16, x16, #0x40\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 14b\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
       "bge 33f\n"
-      "tbz x17, #3, 28f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "tbz x17, #2, 26f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 25f\n"
-      "str d11, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "tbz x8, #3, 28f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "tbz x8, #2, 26f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 25f\n"
+      "str d11, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v11.s }[2], [x17]\n"
       "b 32f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 32f\n"
-      "str s11, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s11, [x17, #0x0]\n"
       "b 32f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 27f\n"
-      "str d10, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "tbz x8, #1, 27f\n"
+      "str d10, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v10.s }[2], [x17]\n"
       "b 32f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 32f\n"
-      "str s10, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s10, [x17, #0x0]\n"
       "b 32f\n"
       "28:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 30f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 29f\n"
-      "str d9, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "tbz x8, #2, 30f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 29f\n"
+      "str d9, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v9.s }[2], [x17]\n"
       "b 32f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 32f\n"
-      "str s9, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s9, [x17, #0x0]\n"
       "b 32f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 31f\n"
-      "str d8, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "tbz x8, #1, 31f\n"
+      "str d8, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v8.s }[2], [x17]\n"
       "b 32f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
       "32:"  // Height 1: Partial direct writeback: Done
       "b 34f\n"
       "33:"  // Height 1: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
       "34:"  // Height 1: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 2b\n"
       "b 206f\n"
       "35:"  // Height 2
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "36:"  // Height 2: Column loop
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x8, #0x10\n"
+      "add x24, x17, x20, LSL #2\n"
       "bge 45f\n"
-      "tbz x17, #3, 40f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x17, #2, 38f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 37f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x8, #3, 40f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x8, #2, 38f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 37f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x17, #1, 39f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x8, #1, 39f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x17, #2, 42f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 41f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x8, #2, 42f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 41f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x17, #1, 43f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x8, #1, 43f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "b 47f\n"
       "46:"  // Height 2: no accumulate
       "movi v8.4s, #0x0\n"
@@ -495,147 +495,147 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "cbnz x14, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "cbnz x15, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
       "50:"  // Height 2: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 53f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
       "ldr d6, [x16, #0x20]\n"
-      "add x12, x12, #0x10\n"
-      "ldr x10, [x16, #0x28]\n"
-      "add x28, x28, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "sub x13, x13, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
-      "ldr x10, [x16, #0x48]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x78]\n"
-      "ldr x27, [x28, #0x8]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x98]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
+      "add x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d1, [x28, #0x0]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
+      "ldr d1, [x9, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x28, [x9, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
+      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr q6, [x16, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
@@ -671,17 +671,17 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x13, 58f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 58f\n"
+      "cmp x14, #0x4\n"
       "blt 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
@@ -693,22 +693,22 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 54b\n"
-      "cbz x13, 58f\n"
       "55:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 56f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "tbz x13, #0, 57f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
+      "cbz x14, 58f\n"
+      "tbz x14, #1, 56f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "tbz x14, #0, 57f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
       "b 57f\n"
       "56:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
       "57:"  // Height 2: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
@@ -720,203 +720,203 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "58:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "bge 67f\n"
-      "tbz x17, #3, 62f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "tbz x17, #2, 60f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 59f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
+      "tbz x8, #3, 62f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "tbz x8, #2, 60f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 59f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
       "b 66f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 66f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
       "b 66f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 61f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
+      "tbz x8, #1, 61f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
       "b 66f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 66f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
       "b 66f\n"
       "62:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 64f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 63f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
+      "tbz x8, #2, 64f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 63f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
       "b 66f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 66f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
       "b 66f\n"
       "64:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 65f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
+      "tbz x8, #1, 65f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
       "b 66f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
       "66:"  // Height 2: Partial direct writeback: Done
       "b 68f\n"
       "67:"  // Height 2: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
       "68:"  // Height 2: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 36b\n"
       "b 206f\n"
       "69:"  // Height 3
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "70:"  // Height 3: Column loop
       "tbz %x[flags], #0, 80f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 79f\n"
-      "tbz x17, #3, 74f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x17, #2, 72f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 71f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x8, #3, 74f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x8, #2, 72f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 71f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 78f\n"
       "71:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 78f\n"
       "72:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x17, #1, 73f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x8, #1, 73f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 78f\n"
       "73:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 78f\n"
       "74:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x17, #2, 76f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 75f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x8, #2, 76f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 75f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 78f\n"
       "75:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 78f\n"
       "76:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x17, #1, 77f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "tbz x8, #1, 77f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 78f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
       "78:"  // Height 3: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 81f\n"
       "79:"  // Height 3: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
       "b 81f\n"
       "80:"  // Height 3: no accumulate
       "movi v8.4s, #0x0\n"
@@ -932,175 +932,175 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "81:"  // Height 3: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "82:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 83f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "cbnz x14, 84f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "cbnz x15, 84f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
       "b 84f\n"
       "83:"  // Height 3: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
       "84:"  // Height 3: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 87f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 86f\n"
       "85:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
-      "ldr x10, [x16, #0x28]\n"
-      "add x12, x12, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x16, #0x38]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x40]\n"
-      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x50]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x60]\n"
-      "cmp x13, #0x20\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xf0]\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
+      "add x27, x27, #0x10\n"
       "add x16, x16, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "mov v0.d[1], x9\n"
-      "ldr d2, [x26, #0x0]\n"
-      "mov v1.d[1], x27\n"
-      "mov v2.d[1], x25\n"
+      "ldr d2, [x27, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x26, [x27, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
+      "mov v2.d[1], x26\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "mov v7.d[1], x11\n"
       "bge 85b\n"
       "86:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
@@ -1151,18 +1151,18 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       "87:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x13, 92f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 92f\n"
+      "cmp x14, #0x4\n"
       "blt 89f\n"
       "88:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
@@ -1178,25 +1178,25 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 88b\n"
-      "cbz x13, 92f\n"
       "89:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 90f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "tbz x13, #0, 91f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
+      "cbz x14, 92f\n"
+      "tbz x14, #1, 90f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "tbz x14, #0, 91f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
       "b 91f\n"
       "90:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
       "91:"  // Height 3: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
@@ -1212,246 +1212,246 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "92:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 82b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "bge 101f\n"
-      "tbz x17, #3, 96f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x17, #2, 94f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 93f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x8, #3, 96f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x8, #2, 94f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 93f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 100f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 100f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 100f\n"
       "94:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 95f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x8, #1, 95f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 100f\n"
       "95:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 100f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 100f\n"
       "96:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 98f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 97f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x8, #2, 98f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 97f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 100f\n"
       "97:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 100f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 100f\n"
       "98:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 99f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x8, #1, 99f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 100f\n"
       "99:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "100:"  // Height 3: Partial direct writeback: Done
       "b 102f\n"
       "101:"  // Height 3: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "102:"  // Height 3: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 70b\n"
       "b 206f\n"
       "103:"  // Height 4
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "104:"  // Height 4: Column loop
       "tbz %x[flags], #0, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 113f\n"
-      "tbz x17, #3, 108f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x17, #2, 106f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 105f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x8, #3, 108f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x8, #2, 106f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 105f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 112f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 112f\n"
       "106:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x17, #1, 107f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x8, #1, 107f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 112f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 112f\n"
       "108:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x17, #2, 110f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 109f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x8, #2, 110f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 109f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 112f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 112f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x17, #1, 111f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x8, #1, 111f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 112f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "112:"  // Height 4: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 115f\n"
       "113:"  // Height 4: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "b 115f\n"
       "114:"  // Height 4: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1471,204 +1471,204 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "115:"  // Height 4: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "116:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 117f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x14, 118f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x15, 118f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
       "b 118f\n"
       "117:"  // Height 4: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
       "118:"  // Height 4: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 121f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 120f\n"
       "119:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x38]\n"
-      "add x12, x12, #0x10\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "add x26, x26, #0x10\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
+      ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
-      "add x24, x24, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "sub x13, x13, #0x10\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "cmp x14, #0x20\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
+      "ldr d3, [x25, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v7.d[1], x11\n"
       "bge 119b\n"
       "120:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
+      ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
@@ -1733,19 +1733,19 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       "121:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x13, 126f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 126f\n"
+      "cmp x14, #0x4\n"
       "blt 123f\n"
       "122:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1765,28 +1765,28 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 122b\n"
-      "cbz x13, 126f\n"
       "123:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 124f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "tbz x13, #0, 125f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
+      "cbz x14, 126f\n"
+      "tbz x14, #1, 124f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "tbz x14, #0, 125f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
       "b 125f\n"
       "124:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
       "125:"  // Height 4: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1806,289 +1806,289 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "126:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 116b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "bge 135f\n"
-      "tbz x17, #3, 130f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "tbz x17, #2, 128f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 127f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "tbz x8, #3, 130f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "tbz x8, #2, 128f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 127f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 134f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 134f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 134f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 129f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "tbz x8, #1, 129f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 134f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 134f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 134f\n"
       "130:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 132f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 131f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "tbz x8, #2, 132f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 131f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 134f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 134f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 134f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 133f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "tbz x8, #1, 133f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 134f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "134:"  // Height 4: Partial direct writeback: Done
       "b 136f\n"
       "135:"  // Height 4: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
       "136:"  // Height 4: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 104b\n"
       "b 206f\n"
       "137:"  // Height 5
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "138:"  // Height 5: Column loop
       "tbz %x[flags], #0, 148f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 147f\n"
-      "tbz x17, #3, 142f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x17, #2, 140f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 139f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x8, #3, 142f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x8, #2, 140f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 139f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 146f\n"
       "139:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x17, #1, 141f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x8, #1, 141f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 146f\n"
       "141:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x17, #2, 144f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 143f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x8, #2, 144f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 143f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 146f\n"
       "143:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x17, #1, 145f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
+      "tbz x8, #1, 145f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
       "b 146f\n"
       "145:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
       "146:"  // Height 5: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 149f\n"
       "147:"  // Height 5: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
       "b 149f\n"
       "148:"  // Height 5: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2112,231 +2112,231 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "149:"  // Height 5: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "150:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 151f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "cbnz x14, 152f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "cbnz x15, 152f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
       "b 152f\n"
       "151:"  // Height 5: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
       "152:"  // Height 5: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 155f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 154f\n"
       "153:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "add x28, x28, #0x10\n"
+      "ldr d6, [x16, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr d6, [x16, #0x20]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x30]\n"
-      "add x26, x26, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "cmp x14, #0x20\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0xa8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
+      "ldr d4, [x23, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v7.d[1], x11\n"
       "bge 153b\n"
       "154:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
@@ -2417,20 +2417,20 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       "155:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x13, 160f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 160f\n"
+      "cmp x14, #0x4\n"
       "blt 157f\n"
       "156:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2454,31 +2454,31 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 156b\n"
-      "cbz x13, 160f\n"
       "157:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 158f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "tbz x13, #0, 159f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "cbz x14, 160f\n"
+      "tbz x14, #1, 158f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "tbz x14, #0, 159f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 159f\n"
       "158:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "159:"  // Height 5: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2502,335 +2502,335 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "160:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 150b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "bge 169f\n"
-      "tbz x17, #3, 164f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x17, #2, 162f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 161f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x8, #3, 164f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x8, #2, 162f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 161f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 168f\n"
       "161:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 168f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 168f\n"
       "162:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 163f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x8, #1, 163f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 168f\n"
       "163:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 168f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 168f\n"
       "164:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 166f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 165f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x8, #2, 166f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 165f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 168f\n"
       "165:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 168f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 168f\n"
       "166:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 167f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x8, #1, 167f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 168f\n"
       "167:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "168:"  // Height 5: Partial direct writeback: Done
       "b 170f\n"
       "169:"  // Height 5: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "170:"  // Height 5: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 138b\n"
       "b 206f\n"
       "171:"  // Height 6
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
-      "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "172:"  // Height 6: Column loop
       "tbz %x[flags], #0, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 181f\n"
-      "tbz x17, #3, 176f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x17, #2, 174f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 173f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x8, #3, 176f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x8, #2, 174f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 173f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 180f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 180f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x17, #1, 175f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x8, #1, 175f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 180f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 180f\n"
       "176:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x17, #2, 178f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 177f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x8, #2, 178f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 177f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 180f\n"
       "177:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 180f\n"
       "178:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x17, #1, 179f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x8, #1, 179f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 180f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "180:"  // Height 6: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 183f\n"
       "181:"  // Height 6: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "b 183f\n"
       "182:"  // Height 6: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2858,260 +2858,260 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "183:"  // Height 6: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "184:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 185f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x14, 186f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x15, 186f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
+      "add x21, x21, x20\n"
       "b 186f\n"
       "185:"  // Height 6: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
-      "add x20, x22, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
+      "add x21, x23, x20\n"
       "186:"  // Height 6: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 189f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 188f\n"
       "187:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x21, #0x8]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "cmp x14, #0x20\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr x19, [x20, #0x8]\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8de  // sdot v30.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
+      "ldr d4, [x23, #0x0]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "ldr d5, [x20, #0x0]\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
-      "mov v5.d[1], x19\n"
+      "ldr d5, [x21, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v5.d[1], x20\n"
+      "mov v7.d[1], x11\n"
       "bge 187b\n"
       "188:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
+      ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x20, x20, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
@@ -3206,21 +3206,21 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
       "189:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x13, 194f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 194f\n"
+      "cmp x14, #0x4\n"
       "blt 191f\n"
       "190:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3248,34 +3248,34 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 190b\n"
-      "cbz x13, 194f\n"
       "191:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 192f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "tbz x13, #0, 193f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "cbz x14, 194f\n"
+      "tbz x14, #1, 192f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "tbz x14, #0, 193f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 193f\n"
       "192:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "193:"  // Height 6: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3303,195 +3303,195 @@ void a64_hybrid_s8s32_dot_6x16_a55 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "194:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 184b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "bge 203f\n"
-      "tbz x17, #3, 198f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "st1 { v29.4s }, [x19], #0x10\n"
-      "tbz x17, #2, 196f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v30.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 195f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "str d31, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
-      "st1 { v31.s }[2], [x19]\n"
+      "tbz x8, #3, 198f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v29.4s }, [x20], #0x10\n"
+      "tbz x8, #2, 196f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v30.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 195f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "str d31, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v31.s }[2], [x20]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 202f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
-      "str s31, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
+      "str s31, [x20, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 197f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "str d30, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
-      "st1 { v30.s }[2], [x19]\n"
+      "tbz x8, #1, 197f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "str d30, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v30.s }[2], [x20]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 202f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
-      "str s30, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
+      "str s30, [x20, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 200f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 199f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "str d29, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
-      "st1 { v29.s }[2], [x19]\n"
+      "tbz x8, #2, 200f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 199f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "str d29, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v29.s }[2], [x20]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 202f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
-      "str s29, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
+      "str s29, [x20, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 201f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
-      "st1 { v28.s }[2], [x19]\n"
+      "tbz x8, #1, 201f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v28.s }[2], [x20]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
-      "str s28, [x19, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
+      "str s28, [x20, #0x0]\n"
       "202:"  // Height 6: Partial direct writeback: Done
       "b 204f\n"
       "203:"  // Height 6: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
-      "str q28, [x19, #0x0]\n"
-      "str q29, [x19, #0x10]\n"
-      "str q30, [x19, #0x20]\n"
-      "str q31, [x19, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
+      "str q28, [x20, #0x0]\n"
+      "str q29, [x20, #0x10]\n"
+      "str q30, [x20, #0x20]\n"
+      "str q31, [x20, #0x30]\n"
       "204:"  // Height 6: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 172b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 206f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 205f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "205:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "206:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp
index e47295a766..ddf776107a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_s8s32_dot_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 69f\n"
       "beq 35f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 12f\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
       "bge 11f\n"
-      "tbz x10, #3, 6f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 4f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 6f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 4f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 3f\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x10, #1, 5f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 5f\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x10, #2, 8f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 7f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 8f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 7f\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s9, [x28, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s9, [x9, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x10, #1, 9f\n"
-      "ldr d8, [x28], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 9f\n"
+      "ldr d8, [x9], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v8.s }[2], [x9]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s8, [x9, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 13f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
       "b 13f\n"
       "12:"  // Height 1: no accumulate
       "movi v8.4s, #0x0\n"
@@ -161,295 +161,295 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
       "13:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "14:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 15f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 16f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 16f\n"
       "15:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "16:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 19f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 18f\n"
       "17:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "add x10, x10, #0x100\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 17b\n"
       "18:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "19:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 24f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 24f\n"
+      "cmp x27, #0x4\n"
       "blt 21f\n"
       "20:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "cmp x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "cmp x27, #0x4\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x9, x9, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x10, x10, #0x40\n"
       "bge 20b\n"
-      "cbz x26, 24f\n"
       "21:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 22f\n"
-      "ldr h0, [x25], #0x2\n"
-      "tbz x26, #0, 23f\n"
-      "ld1 { v0.b }[2], [x25]\n"
+      "cbz x27, 24f\n"
+      "tbz x27, #1, 22f\n"
+      "ldr h0, [x26], #0x2\n"
+      "tbz x27, #0, 23f\n"
+      "ld1 { v0.b }[2], [x26]\n"
       "b 23f\n"
       "22:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x20]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x10, x10, #0x40\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 14b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "bge 33f\n"
-      "tbz x10, #3, 28f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 26f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 25f\n"
-      "str d11, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 28f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 26f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 25f\n"
+      "str d11, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 32f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 32f\n"
-      "str s11, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s11, [x9, #0x0]\n"
       "b 32f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 27f\n"
-      "str d10, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 27f\n"
+      "str d10, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 32f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 32f\n"
-      "str s10, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s10, [x9, #0x0]\n"
       "b 32f\n"
       "28:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 30f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 29f\n"
-      "str d9, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 30f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 29f\n"
+      "str d9, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 32f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 32f\n"
-      "str s9, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s9, [x9, #0x0]\n"
       "b 32f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 31f\n"
-      "str d8, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 31f\n"
+      "str d8, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 32f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "32:"  // Height 1: Partial direct writeback: Done
       "b 34f\n"
       "33:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "34:"  // Height 1: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 206f\n"
       "35:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "36:"  // Height 2: Column loop
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
+      "add x24, x9, x20, LSL #2\n"
       "bge 45f\n"
-      "tbz x10, #3, 40f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 38f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 37f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 40f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 38f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 37f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x10, #1, 39f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 39f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x10, #2, 42f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 41f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 42f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 41f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x10, #1, 43f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 43f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "b 47f\n"
       "46:"  // Height 2: no accumulate
       "movi v8.4s, #0x0\n"
@@ -461,392 +461,392 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "50:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 53f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 58f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 58f\n"
+      "cmp x27, #0x4\n"
       "blt 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 54b\n"
-      "cbz x26, 58f\n"
       "55:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 56f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "tbz x26, #0, 57f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
+      "cbz x27, 58f\n"
+      "tbz x27, #1, 56f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "tbz x27, #0, 57f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
       "b 57f\n"
       "56:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
       "57:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       "58:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "bge 67f\n"
-      "tbz x10, #3, 62f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 60f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 59f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 62f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 60f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 59f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
       "b 66f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 66f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
       "b 66f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 61f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 61f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
       "b 66f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 66f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
       "b 66f\n"
       "62:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 64f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 63f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 64f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 63f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
       "b 66f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 66f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
       "b 66f\n"
       "64:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 65f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 65f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
       "b 66f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
       "66:"  // Height 2: Partial direct writeback: Done
       "b 68f\n"
       "67:"  // Height 2: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
       "68:"  // Height 2: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 36b\n"
       "b 206f\n"
       "69:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "70:"  // Height 3: Column loop
       "tbz %x[flags], #0, 80f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 79f\n"
-      "tbz x10, #3, 74f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 72f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 71f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 74f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 72f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 71f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 78f\n"
       "71:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 78f\n"
       "72:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x10, #1, 73f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 73f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 78f\n"
       "73:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 78f\n"
       "74:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x10, #2, 76f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 75f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 76f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 75f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 78f\n"
       "75:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 78f\n"
       "76:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x10, #1, 77f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 77f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 78f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
       "78:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 81f\n"
       "79:"  // Height 3: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
       "b 81f\n"
       "80:"  // Height 3: no accumulate
       "movi v8.4s, #0x0\n"
@@ -862,180 +862,180 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "81:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "82:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 83f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 84f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 84f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 84f\n"
       "83:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "84:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 87f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 86f\n"
       "85:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 85b\n"
       "86:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -1043,26 +1043,26 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       "87:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 92f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 92f\n"
+      "cmp x27, #0x4\n"
       "blt 89f\n"
       "88:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1070,33 +1070,33 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 88b\n"
-      "cbz x26, 92f\n"
       "89:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 90f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "tbz x26, #0, 91f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
+      "cbz x27, 92f\n"
+      "tbz x27, #1, 90f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "tbz x27, #0, 91f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
       "b 91f\n"
       "90:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
       "91:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1104,246 +1104,246 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       "92:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 82b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "bge 101f\n"
-      "tbz x10, #3, 96f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 94f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 93f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 96f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 94f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 93f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 100f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 100f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 100f\n"
       "94:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 95f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 95f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 100f\n"
       "95:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 100f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 100f\n"
       "96:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 98f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 97f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 98f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 97f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 100f\n"
       "97:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 100f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 100f\n"
       "98:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 99f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 99f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 100f\n"
       "99:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "100:"  // Height 3: Partial direct writeback: Done
       "b 102f\n"
       "101:"  // Height 3: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "102:"  // Height 3: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 70b\n"
       "b 206f\n"
       "103:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "104:"  // Height 4: Column loop
       "tbz %x[flags], #0, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 113f\n"
-      "tbz x10, #3, 108f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 106f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 105f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 108f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 106f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 105f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 112f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 112f\n"
       "106:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x10, #1, 107f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 107f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 112f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 112f\n"
       "108:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x10, #2, 110f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 109f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 110f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 109f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 112f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 112f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x10, #1, 111f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 111f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 112f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "112:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 115f\n"
       "113:"  // Height 4: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "b 115f\n"
       "114:"  // Height 4: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1363,219 +1363,219 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "115:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "116:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 117f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 118f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 118f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 118f\n"
       "117:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "118:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 121f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 120f\n"
       "119:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 119b\n"
       "120:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -1585,29 +1585,29 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       "121:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 126f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 126f\n"
+      "cmp x27, #0x4\n"
       "blt 123f\n"
       "122:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1617,38 +1617,38 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 122b\n"
-      "cbz x26, 126f\n"
       "123:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 124f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "tbz x26, #0, 125f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
+      "cbz x27, 126f\n"
+      "tbz x27, #1, 124f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "tbz x27, #0, 125f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
       "b 125f\n"
       "124:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
       "125:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1658,289 +1658,289 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       "126:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 116b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "bge 135f\n"
-      "tbz x10, #3, 130f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 128f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 127f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 130f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 128f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 127f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 134f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 134f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 134f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 129f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 129f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 134f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 134f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 134f\n"
       "130:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 132f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 131f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 132f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 131f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 134f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 134f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 134f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 133f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 133f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 134f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "134:"  // Height 4: Partial direct writeback: Done
       "b 136f\n"
       "135:"  // Height 4: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
       "136:"  // Height 4: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 104b\n"
       "b 206f\n"
       "137:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "138:"  // Height 5: Column loop
       "tbz %x[flags], #0, 148f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 147f\n"
-      "tbz x10, #3, 142f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 140f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 139f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 142f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 140f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 139f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 146f\n"
       "139:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x10, #1, 141f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 141f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 146f\n"
       "141:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x10, #2, 144f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 143f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 144f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 143f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 146f\n"
       "143:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x10, #1, 145f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 145f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
       "b 146f\n"
       "145:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
       "146:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 149f\n"
       "147:"  // Height 5: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
       "b 149f\n"
       "148:"  // Height 5: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1964,258 +1964,258 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "149:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "150:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 151f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 152f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 152f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 152f\n"
       "151:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "152:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 155f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 154f\n"
       "153:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 153b\n"
       "154:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2227,32 +2227,32 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       "155:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 160f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 160f\n"
+      "cmp x27, #0x4\n"
       "blt 157f\n"
       "156:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2264,43 +2264,43 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 156b\n"
-      "cbz x26, 160f\n"
       "157:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 158f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "tbz x26, #0, 159f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
+      "cbz x27, 160f\n"
+      "tbz x27, #1, 158f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "tbz x27, #0, 159f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
       "b 159f\n"
       "158:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
       "159:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2312,335 +2312,335 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       "160:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 150b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "cmp x11, #0x10\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "bge 169f\n"
-      "tbz x10, #3, 164f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 162f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 161f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 164f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 162f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 161f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 168f\n"
       "161:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 168f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 168f\n"
       "162:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 163f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 163f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 168f\n"
       "163:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 168f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 168f\n"
       "164:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 166f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 165f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 166f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 165f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 168f\n"
       "165:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 168f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 168f\n"
       "166:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 167f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 167f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 168f\n"
       "167:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "168:"  // Height 5: Partial direct writeback: Done
       "b 170f\n"
       "169:"  // Height 5: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "170:"  // Height 5: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 138b\n"
       "b 206f\n"
       "171:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "172:"  // Height 6: Column loop
       "tbz %x[flags], #0, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 181f\n"
-      "tbz x10, #3, 176f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 174f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 173f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 176f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 174f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 173f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 180f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 180f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x10, #1, 175f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 175f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 180f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 180f\n"
       "176:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x10, #2, 178f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 177f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 178f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 177f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 180f\n"
       "177:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 180f\n"
       "178:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x10, #1, 179f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 179f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 180f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "180:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 183f\n"
       "181:"  // Height 6: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "b 183f\n"
       "182:"  // Height 6: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2668,297 +2668,297 @@ void a64_hybrid_s8s32_dot_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "183:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "184:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 185f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 186f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 186f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 186f\n"
       "185:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "186:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 189f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 188f\n"
       "187:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d6  // sdot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8da  // sdot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8de  // sdot v30.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x4fa0e8eb  // sdot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x4fa1e8ef  // sdot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x4fa2e8f3  // sdot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x4fa3e8f7  // sdot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 187b\n"
       "188:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d6  // sdot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4f84e0da  // sdot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0de  // sdot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4f80e0eb  // sdot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ef  // sdot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f3  // sdot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f7  // sdot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4fa0e0c8  // sdot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0cc  // sdot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d0  // sdot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d4  // sdot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0d8  // sdot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0dc  // sdot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4fa0e0e9  // sdot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ed  // sdot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f1  // sdot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f5  // sdot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0f9  // sdot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0fd  // sdot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4fa0e0ca  // sdot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ce  // sdot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0d2  // sdot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0d6  // sdot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0da  // sdot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0de  // sdot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x4fa0e0eb  // sdot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x4fa1e0ef  // sdot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x4fa2e0f3  // sdot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x4fa3e0f7  // sdot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x4fa4e0fb  // sdot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x4fa5e0ff  // sdot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x4f80e8c8  // sdot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8cc  // sdot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d0  // sdot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d4  // sdot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8d8  // sdot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8dc  // sdot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x4f80e8e9  // sdot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ed  // sdot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f1  // sdot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f5  // sdot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8f9  // sdot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8fd  // sdot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x4f80e8ca  // sdot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ce  // sdot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x4f82e8d2  // sdot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x4f83e8d6  // sdot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x4f84e8da  // sdot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x4f85e8de  // sdot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x4f80e8eb  // sdot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x4f81e8ef  // sdot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x4f82e8f3  // sdot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x4f83e8f7  // sdot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x4f84e8fb  // sdot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x4f85e8ff  // sdot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x4fa0e8c8  // sdot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8cc  // sdot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d0  // sdot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8d4  // sdot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8d8  // sdot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8dc  // sdot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x4fa0e8e9  // sdot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ed  // sdot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8f1  // sdot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x4fa3e8f5  // sdot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x4fa4e8f9  // sdot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8fd  // sdot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4fa0e8ca  // sdot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x4fa1e8ce  // sdot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x4fa2e8d2  // sdot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2972,35 +2972,35 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4fa4e8fb  // sdot v27.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x4fa5e8ff  // sdot v31.4s, v7.16b, v5.4b[3]\n"
       "189:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 194f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 194f\n"
+      "cmp x27, #0x4\n"
       "blt 191f\n"
       "190:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3014,48 +3014,48 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 190b\n"
-      "cbz x26, 194f\n"
       "191:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 192f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "tbz x26, #0, 193f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "cbz x27, 194f\n"
+      "tbz x27, #1, 192f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "tbz x27, #0, 193f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 193f\n"
       "192:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "193:"  // Height 6: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4f80e0c8  // sdot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4f81e0cc  // sdot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d0  // sdot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x4f83e0d4  // sdot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x4f84e0d8  // sdot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x4f85e0dc  // sdot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4f80e0e9  // sdot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ed  // sdot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x4f82e0f1  // sdot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x4f83e0f5  // sdot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x4f84e0f9  // sdot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0fd  // sdot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x4f80e0ca  // sdot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x4f81e0ce  // sdot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x4f82e0d2  // sdot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3069,195 +3069,195 @@ void a64_hybrid_s8s32_dot_6x16 (
       ".inst 0x4f84e0fb  // sdot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x4f85e0ff  // sdot v31.4s, v7.16b, v5.4b[0]\n"
       "194:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 184b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "bge 203f\n"
-      "tbz x10, #3, 198f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "st1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 196f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 195f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "str d31, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
-      "st1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 198f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 196f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 195f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "str d31, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v31.s }[2], [x20]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 202f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
-      "str s31, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
+      "str s31, [x20, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 197f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "str d30, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
-      "st1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 197f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "str d30, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v30.s }[2], [x20]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 202f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
-      "str s30, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
+      "str s30, [x20, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 200f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 199f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "str d29, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
-      "st1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 200f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 199f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "str d29, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v29.s }[2], [x20]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 202f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
-      "str s29, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
+      "str s29, [x20, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 201f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
-      "st1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 201f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v28.s }[2], [x20]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
-      "str s28, [x19, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
+      "str s28, [x20, #0x0]\n"
       "202:"  // Height 6: Partial direct writeback: Done
       "b 204f\n"
       "203:"  // Height 6: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
-      "str q28, [x19, #0x0]\n"
-      "str q29, [x19, #0x10]\n"
-      "str q30, [x19, #0x20]\n"
-      "str q31, [x19, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
+      "str q28, [x20, #0x0]\n"
+      "str q29, [x20, #0x10]\n"
+      "str q30, [x20, #0x20]\n"
+      "str q31, [x20, #0x30]\n"
       "204:"  // Height 6: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 172b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 206f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 205f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "205:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "206:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp
index a9f6b06ae1..f48623e129 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 75f\n"
       "beq 38f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 13f\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
       "bge 11f\n"
-      "tbz x10, #3, 6f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 4f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v16.s }[2], [x28]\n"
+      "tbz x11, #3, 6f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 4f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 3f\n"
+      "ldr d16, [x9], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v16.s }[2], [x9]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s16, [x28, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s16, [x9, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x10, #1, 5f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "tbz x11, #1, 5f\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x10, #2, 8f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 7f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "tbz x11, #2, 8f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 7f\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x10, #1, 9f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "tbz x11, #1, 9f\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s9, [x9, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 12f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
       "12:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -174,337 +174,333 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "14:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 17f\n"
       "16:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "17:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 20f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 19f\n"
       "18:"  // Height 1: Multiply loop: Main loop head
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 18b\n"
       "19:"  // Height 1: Multiply loop: Single iteration only
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "20:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 27f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 27f\n"
+      "cmp x27, #0x8\n"
       "blt 22f\n"
       "21:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       "bge 21b\n"
-      "cbz x26, 27f\n"
       "22:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 24f\n"
-      "ldr s1, [x25], #0x4\n"
-      "tbz x26, #1, 23f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[6], [x25]\n"
+      "cbz x27, 27f\n"
+      "tbz x27, #2, 24f\n"
+      "ldr s1, [x26], #0x4\n"
+      "tbz x27, #1, 23f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[6], [x26]\n"
       "b 26f\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[4], [x25]\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[4], [x26]\n"
       "b 26f\n"
       "24:"  // Height 1: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 25f\n"
-      "ldr h1, [x25], #0x2\n"
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[2], [x25]\n"
+      "tbz x27, #1, 25f\n"
+      "ldr h1, [x26], #0x2\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[2], [x26]\n"
       "b 26f\n"
       "25:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
       "26:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       "27:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 15b\n"
+      "cmp x11, #0x10\n"
       "uzp1 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
-      "cmp x10, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
       "bge 36f\n"
-      "tbz x10, #3, 31f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 29f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 28f\n"
-      "str d11, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 31f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 29f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 28f\n"
+      "str d11, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 35f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 35f\n"
-      "str s11, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s11, [x9, #0x0]\n"
       "b 35f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 30f\n"
-      "str d10, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 30f\n"
+      "str d10, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 35f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 35f\n"
-      "str s10, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s10, [x9, #0x0]\n"
       "b 35f\n"
       "31:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 33f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 32f\n"
-      "str d9, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 33f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 32f\n"
+      "str d9, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 35f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 35f\n"
-      "str s9, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s9, [x9, #0x0]\n"
       "b 35f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 34f\n"
-      "str d8, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 34f\n"
+      "str d8, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 35f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "35:"  // Height 1: Partial direct writeback: Done
       "b 37f\n"
       "36:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "37:"  // Height 1: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 224f\n"
       "38:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "39:"  // Height 2: Column loop
       "tbz %x[flags], #0, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
+      "add x24, x9, x20, LSL #2\n"
       "bge 48f\n"
-      "tbz x10, #3, 43f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 41f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 40f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 43f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 41f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 40f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 47f\n"
       "40:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 47f\n"
       "41:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x10, #1, 42f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 42f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 47f\n"
       "42:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 47f\n"
       "43:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x10, #2, 45f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 44f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 45f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 44f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 47f\n"
       "44:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 47f\n"
       "45:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x10, #1, 46f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 46f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 47f\n"
       "46:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
       "47:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 49f\n"
       "48:"  // Height 2: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "49:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -525,398 +521,398 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "51:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "52:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 53f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 54f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 54f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 54f\n"
       "53:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "54:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 57f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 56f\n"
       "55:"  // Height 2: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "bge 55b\n"
       "56:"  // Height 2: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "57:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 64f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 64f\n"
+      "cmp x27, #0x8\n"
       "blt 59f\n"
       "58:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x20]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x60]\n"
+      "ldr q7, [x10, #0x70]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       "bge 58b\n"
-      "cbz x26, 64f\n"
       "59:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 61f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "tbz x26, #1, 60f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
+      "cbz x27, 64f\n"
+      "tbz x27, #2, 61f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "tbz x27, #1, 60f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
       "b 63f\n"
       "60:"  // Height 2: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
       "b 63f\n"
       "61:"  // Height 2: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 62f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
+      "tbz x27, #1, 62f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
       "b 63f\n"
       "62:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
       "63:"  // Height 2: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       "64:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 52b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "bge 73f\n"
-      "tbz x10, #3, 68f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 66f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 65f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
+      "tbz x11, #3, 68f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 66f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 65f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
       "b 72f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 72f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
       "b 72f\n"
       "66:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 67f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
+      "tbz x11, #1, 67f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
       "b 72f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 72f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
       "b 72f\n"
       "68:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 70f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 69f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
+      "tbz x11, #2, 70f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 69f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
       "b 72f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 72f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
       "b 72f\n"
       "70:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 71f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
+      "tbz x11, #1, 71f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
       "b 72f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
       "72:"  // Height 2: Partial direct writeback: Done
       "b 74f\n"
       "73:"  // Height 2: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
       "74:"  // Height 2: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 39b\n"
       "b 224f\n"
       "75:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "76:"  // Height 3: Column loop
       "tbz %x[flags], #0, 87f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 85f\n"
-      "tbz x10, #3, 80f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 78f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 77f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
+      "tbz x11, #3, 80f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 78f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 77f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
       "b 84f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
       "b 84f\n"
       "78:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x10, #1, 79f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x11, #1, 79f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 84f\n"
       "79:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 84f\n"
       "80:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x10, #2, 82f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 81f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x11, #2, 82f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 81f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 84f\n"
       "81:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 84f\n"
       "82:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x10, #1, 83f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x11, #1, 83f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 84f\n"
       "83:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
       "84:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 86f\n"
       "85:"  // Height 3: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
       "86:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -953,285 +949,281 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "88:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "89:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 90f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 91f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 91f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 91f\n"
       "90:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "91:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 94f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 93f\n"
       "92:"  // Height 3: Multiply loop: Main loop head
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 92b\n"
       "93:"  // Height 3: Multiply loop: Single iteration only
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       "94:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 101f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 101f\n"
+      "cmp x27, #0x8\n"
       "blt 96f\n"
       "95:"  // Height 3: Multiply loop: Odd block loop
-      "movi v4.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       "bge 95b\n"
-      "cbz x26, 101f\n"
       "96:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 98f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "tbz x26, #1, 97f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
+      "cbz x27, 101f\n"
+      "tbz x27, #2, 98f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "tbz x27, #1, 97f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
       "b 100f\n"
       "97:"  // Height 3: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
       "b 100f\n"
       "98:"  // Height 3: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 99f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
+      "tbz x27, #1, 99f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
       "b 100f\n"
       "99:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
       "100:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v4.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       "101:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 89b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v16.2d, v16.2d, v20.2d\n"
@@ -1239,234 +1231,234 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "bge 110f\n"
-      "tbz x10, #3, 105f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 103f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 102f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 105f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 103f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 102f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 109f\n"
       "102:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 109f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 109f\n"
       "103:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 104f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 104f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 109f\n"
       "104:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 109f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 109f\n"
       "105:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 107f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 106f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 107f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 106f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 109f\n"
       "106:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 109f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 109f\n"
       "107:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 108f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 108f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 109f\n"
       "108:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "109:"  // Height 3: Partial direct writeback: Done
       "b 111f\n"
       "110:"  // Height 3: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "111:"  // Height 3: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 76b\n"
       "b 224f\n"
       "112:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "113:"  // Height 4: Column loop
       "tbz %x[flags], #0, 124f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 122f\n"
-      "tbz x10, #3, 117f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 115f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 114f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 117f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 115f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 114f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 121f\n"
       "114:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 121f\n"
       "115:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x10, #1, 116f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 116f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 121f\n"
       "116:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 121f\n"
       "117:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x10, #2, 119f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 118f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 119f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 118f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 121f\n"
       "118:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 121f\n"
       "119:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x10, #1, 120f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 120f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 121f\n"
       "120:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "121:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 123f\n"
       "122:"  // Height 4: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "123:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1503,301 +1495,301 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "125:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "126:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 127f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 128f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 128f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 128f\n"
       "127:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "128:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 131f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 130f\n"
       "129:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 129b\n"
       "130:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       "131:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 138f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 138f\n"
+      "cmp x27, #0x8\n"
       "blt 133f\n"
       "132:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
+      "cmp x27, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       "bge 132b\n"
-      "cbz x26, 138f\n"
       "133:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 135f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "tbz x26, #1, 134f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
+      "cbz x27, 138f\n"
+      "tbz x27, #2, 135f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "tbz x27, #1, 134f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
       "b 137f\n"
       "134:"  // Height 4: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
       "b 137f\n"
       "135:"  // Height 4: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 136f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "tbz x27, #1, 136f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 137f\n"
       "136:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "137:"  // Height 4: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       "138:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 126b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
@@ -1808,275 +1800,275 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "bge 147f\n"
-      "tbz x10, #3, 142f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 140f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 139f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
+      "tbz x11, #3, 142f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 140f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 139f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
       "b 146f\n"
       "139:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 146f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 141f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
+      "tbz x11, #1, 141f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
       "b 146f\n"
       "141:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 146f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 144f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 143f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
+      "tbz x11, #2, 144f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 143f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
       "b 146f\n"
       "143:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 146f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 145f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
+      "tbz x11, #1, 145f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
       "b 146f\n"
       "145:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
       "146:"  // Height 4: Partial direct writeback: Done
       "b 148f\n"
       "147:"  // Height 4: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
       "148:"  // Height 4: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 113b\n"
       "b 224f\n"
       "149:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "150:"  // Height 5: Column loop
       "tbz %x[flags], #0, 161f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 159f\n"
-      "tbz x10, #3, 154f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 152f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 151f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d6, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v6.s }[2], [x20]\n"
+      "tbz x11, #3, 154f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 152f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 151f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d6, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v6.s }[2], [x21]\n"
       "b 158f\n"
       "151:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s6, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s6, [x21, #0x0]\n"
       "b 158f\n"
       "152:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x10, #1, 153f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x11, #1, 153f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 158f\n"
       "153:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 158f\n"
       "154:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x10, #2, 156f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 155f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x11, #2, 156f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 155f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 158f\n"
       "155:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 158f\n"
       "156:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x10, #1, 157f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x11, #1, 157f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 158f\n"
       "157:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "158:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 160f\n"
       "159:"  // Height 5: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q25, [x20, #0x0]\n"
-      "ldr q26, [x20, #0x10]\n"
-      "ldr q27, [x20, #0x20]\n"
-      "ldr q6, [x20, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q25, [x21, #0x0]\n"
+      "ldr q26, [x21, #0x10]\n"
+      "ldr q27, [x21, #0x20]\n"
+      "ldr q6, [x21, #0x30]\n"
       "160:"  // Height 5: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2129,212 +2121,210 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "162:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "163:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 164f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 165f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 165f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 165f\n"
       "164:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "165:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 168f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 167f\n"
       "166:"  // Height 5: Multiply loop: Main loop head
-      "movi v6.4s, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "cmp x26, #0x20\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
       "bge 166b\n"
       "167:"  // Height 5: Multiply loop: Single iteration only
-      "movi v6.4s, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
@@ -2342,136 +2332,134 @@ void a64_hybrid_s8s32_mmla_6x16 (
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
       "168:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 175f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 175f\n"
+      "cmp x27, #0x8\n"
       "blt 170f\n"
       "169:"  // Height 5: Multiply loop: Odd block loop
-      "movi v7.4s, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr q6, [x9, #0x0]\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a498  // smmla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x4e87a49c  // smmla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a499  // smmla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49d  // smmla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49a  // smmla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49e  // smmla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49b  // smmla v27.4s, v4.16b, v6.16b\n"
       ".inst 0x4e87a40f  // smmla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49f  // smmla v31.4s, v4.16b, v7.16b\n"
       "bge 169b\n"
-      "cbz x26, 175f\n"
       "170:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 172f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "tbz x26, #1, 171f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
+      "cbz x27, 175f\n"
+      "tbz x27, #2, 172f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "tbz x27, #1, 171f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
       "b 174f\n"
       "171:"  // Height 5: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
       "b 174f\n"
       "172:"  // Height 5: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 173f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
+      "tbz x27, #1, 173f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
       "b 174f\n"
       "173:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
       "174:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "movi v6.4s, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
@@ -2479,30 +2467,30 @@ void a64_hybrid_s8s32_mmla_6x16 (
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
       "175:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 163b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
@@ -2515,319 +2503,319 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "uzp1 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "bge 184f\n"
-      "tbz x10, #3, 179f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 177f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 176f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 179f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 177f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 176f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 183f\n"
       "176:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 183f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 183f\n"
       "177:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 178f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 178f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 183f\n"
       "178:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 183f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 183f\n"
       "179:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 181f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 180f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 181f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 180f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 183f\n"
       "180:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 183f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 183f\n"
       "181:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 182f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 182f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 183f\n"
       "182:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "183:"  // Height 5: Partial direct writeback: Done
       "b 185f\n"
       "184:"  // Height 5: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "185:"  // Height 5: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 150b\n"
       "b 224f\n"
       "186:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "187:"  // Height 6: Column loop
       "tbz %x[flags], #0, 198f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 196f\n"
-      "tbz x10, #3, 191f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 189f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 188f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d6, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v6.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 191f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 189f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 188f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d6, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v6.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 195f\n"
       "188:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s6, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s6, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 195f\n"
       "189:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x10, #1, 190f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 190f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 195f\n"
       "190:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 195f\n"
       "191:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x10, #2, 193f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 192f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 193f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 192f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 195f\n"
       "192:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 195f\n"
       "193:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x10, #1, 194f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 194f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 195f\n"
       "194:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "195:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 197f\n"
       "196:"  // Height 6: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q25, [x20, #0x0]\n"
-      "ldr q26, [x20, #0x10]\n"
-      "ldr q27, [x20, #0x20]\n"
-      "ldr q6, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q25, [x21, #0x0]\n"
+      "ldr q26, [x21, #0x10]\n"
+      "ldr q27, [x21, #0x20]\n"
+      "ldr q6, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "197:"  // Height 6: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2880,219 +2868,219 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "199:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "200:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 201f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 202f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 202f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 202f\n"
       "201:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "202:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 205f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 204f\n"
       "203:"  // Height 6: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 203b\n"
       "204:"  // Height 6: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x4e87a428  // smmla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a470  // smmla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b8  // smmla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x4e86a42c  // smmla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a474  // smmla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bc  // smmla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x4e87a429  // smmla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a471  // smmla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4b9  // smmla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x4e86a42d  // smmla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a475  // smmla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bd  // smmla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x4e87a42a  // smmla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a472  // smmla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4ba  // smmla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e86a476  // smmla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4be  // smmla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x4e87a42b  // smmla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x4e87a473  // smmla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x4e87a4bb  // smmla v27.4s, v5.16b, v7.16b\n"
@@ -3100,48 +3088,48 @@ void a64_hybrid_s8s32_mmla_6x16 (
       ".inst 0x4e86a477  // smmla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x4e86a4bf  // smmla v31.4s, v5.16b, v6.16b\n"
       "205:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 212f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 212f\n"
+      "cmp x27, #0x8\n"
       "blt 207f\n"
       "206:"  // Height 6: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "cmp x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a450  // smmla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a498  // smmla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a454  // smmla v20.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49c  // smmla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a451  // smmla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a499  // smmla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a455  // smmla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49d  // smmla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a452  // smmla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49a  // smmla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x4e87a40e  // smmla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a456  // smmla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49e  // smmla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e86a40b  // smmla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a453  // smmla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49b  // smmla v27.4s, v4.16b, v6.16b\n"
@@ -3149,128 +3137,128 @@ void a64_hybrid_s8s32_mmla_6x16 (
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49f  // smmla v31.4s, v4.16b, v7.16b\n"
       "bge 206b\n"
-      "cbz x26, 212f\n"
       "207:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 209f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "ldr s6, [x20], #0x4\n"
-      "tbz x26, #1, 208f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "ld1 { v6.h }[2], [x20], #0x2\n"
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
-      "ld1 { v6.b }[6], [x20]\n"
+      "cbz x27, 212f\n"
+      "tbz x27, #2, 209f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "ldr s6, [x21], #0x4\n"
+      "tbz x27, #1, 208f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "ld1 { v6.h }[2], [x21], #0x2\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
+      "ld1 { v6.b }[6], [x21]\n"
       "b 211f\n"
       "208:"  // Height 6: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
-      "ld1 { v6.b }[4], [x20]\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
+      "ld1 { v6.b }[4], [x21]\n"
       "b 211f\n"
       "209:"  // Height 6: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 210f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "ldr h6, [x20], #0x2\n"
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
-      "ld1 { v6.b }[2], [x20]\n"
+      "tbz x27, #1, 210f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "ldr h6, [x21], #0x2\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
+      "ld1 { v6.b }[2], [x21]\n"
       "b 211f\n"
       "210:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
-      "ldr b6, [x20, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
+      "ldr b6, [x21, #0x0]\n"
       "211:"  // Height 6: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x4e87a408  // smmla v8.4s, v0.16b, v7.16b\n"
+      "trn1 v4.2d, v5.2d, v6.2d\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x4e87a450  // smmla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a498  // smmla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x4e86a40c  // smmla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49c  // smmla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x4e87a409  // smmla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a451  // smmla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a499  // smmla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x4e86a40d  // smmla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49d  // smmla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x4e87a40a  // smmla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x4e87a452  // smmla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49a  // smmla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x4e86a40e  // smmla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49e  // smmla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x4e87a453  // smmla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x4e87a49b  // smmla v27.4s, v4.16b, v7.16b\n"
       ".inst 0x4e86a40f  // smmla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x4e86a457  // smmla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x4e86a49f  // smmla v31.4s, v4.16b, v6.16b\n"
       "212:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 200b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
@@ -3285,177 +3273,177 @@ void a64_hybrid_s8s32_mmla_6x16 (
       "uzp1 v30.2d, v27.2d, v31.2d\n"
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "bge 221f\n"
-      "tbz x10, #3, 216f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v24.4s }, [x19], #0x10\n"
-      "st1 { v25.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 214f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
-      "st1 { v26.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 213f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
-      "str d27, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
-      "st1 { v27.s }[2], [x19]\n"
+      "tbz x11, #3, 216f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v24.4s }, [x20], #0x10\n"
+      "st1 { v25.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 214f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
+      "st1 { v26.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 213f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
+      "str d27, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
+      "st1 { v27.s }[2], [x20]\n"
       "b 220f\n"
       "213:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 220f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
-      "str s27, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
+      "str s27, [x20, #0x0]\n"
       "b 220f\n"
       "214:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 215f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
-      "str d26, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
-      "st1 { v26.s }[2], [x19]\n"
+      "tbz x11, #1, 215f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
+      "str d26, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
+      "st1 { v26.s }[2], [x20]\n"
       "b 220f\n"
       "215:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 220f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
-      "str s26, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
+      "str s26, [x20, #0x0]\n"
       "b 220f\n"
       "216:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 218f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
-      "st1 { v24.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 217f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
-      "str d25, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
-      "st1 { v25.s }[2], [x19]\n"
+      "tbz x11, #2, 218f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v24.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 217f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
+      "str d25, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
+      "st1 { v25.s }[2], [x20]\n"
       "b 220f\n"
       "217:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 220f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
-      "str s25, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
+      "str s25, [x20, #0x0]\n"
       "b 220f\n"
       "218:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 219f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "str d24, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
-      "st1 { v23.s }[2], [x20]\n"
-      "st1 { v24.s }[2], [x19]\n"
+      "tbz x11, #1, 219f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "str d24, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v23.s }[2], [x21]\n"
+      "st1 { v24.s }[2], [x20]\n"
       "b 220f\n"
       "219:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
-      "str s23, [x20, #0x0]\n"
-      "str s24, [x19, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
+      "str s23, [x21, #0x0]\n"
+      "str s24, [x20, #0x0]\n"
       "220:"  // Height 6: Partial direct writeback: Done
       "b 222f\n"
       "221:"  // Height 6: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
-      "str q23, [x20, #0x0]\n"
-      "str q28, [x20, #0x10]\n"
-      "str q29, [x20, #0x20]\n"
-      "str q30, [x20, #0x30]\n"
-      "str q24, [x19, #0x0]\n"
-      "str q25, [x19, #0x10]\n"
-      "str q26, [x19, #0x20]\n"
-      "str q27, [x19, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
+      "str q23, [x21, #0x0]\n"
+      "str q28, [x21, #0x10]\n"
+      "str q29, [x21, #0x20]\n"
+      "str q30, [x21, #0x30]\n"
+      "str q24, [x20, #0x0]\n"
+      "str q25, [x20, #0x10]\n"
+      "str q26, [x20, #0x20]\n"
+      "str q27, [x20, #0x30]\n"
       "222:"  // Height 6: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 187b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 224f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 223f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "223:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "224:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
index c410374357..b9caf545f1 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,232 +85,220 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 61f\n"
       "beq 31f\n"
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
       "movi v15.16b, #0x1\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x14, %x[output_ptr]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "cbnz x12, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x10, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 11f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
-      "ldr x24, [x10, #0x18]\n"
-      "add x25, x25, #0x10\n"
-      "ldr d6, [x10, #0x20]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
+      "ldr d4, [x13, #0x70]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "mov v7.d[1], x19\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
-      "ldr x19, [x10, #0x58]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr d7, [x13, #0xa0]\n"
+      "mov v4.d[1], x9\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v9.d[1], x19\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
-      "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
-      "ldr d6, [x10, #0x90]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d4, [x13, #0xe0]\n"
+      "mov v5.d[1], x28\n"
+      "ldr x27, [x13, #0x98]\n"
+      "mov v6.d[1], x27\n"
+      "ldr x26, [x13, #0xa8]\n"
+      "mov v7.d[1], x26\n"
+      "ldr x25, [x13, #0xb8]\n"
+      "mov v8.d[1], x25\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "mov v6.d[1], x23\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
-      "ldr x23, [x10, #0xb8]\n"
-      "mov v7.d[1], x19\n"
-      "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
-      "ldr x23, [x10, #0xd8]\n"
+      "ldr x28, [x13, #0xf8]\n"
+      "mov v9.d[1], x24\n"
+      "mov v10.d[1], x20\n"
+      "add x10, x10, #0x10\n"
+      "mov v4.d[1], x9\n"
+      "add x13, x13, #0x100\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr d4, [x10, #0xe0]\n"
-      "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
-      "ldr d5, [x10, #0xf0]\n"
-      "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 8f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      "ldr q0, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q4, [x13, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
       "bge 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
-      "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x10\n"
-      "ldr q7, [x10, #0x30]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q4, [x10, #0x70]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "ldr q5, [x10, #0x80]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 10f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 18f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 18f\n"
+      "cmp x11, #0x4\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       "bge 12b\n"
-      "cbz x26, 18f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 15f\n"
-      "ldr h0, [x25], #0x2\n"
-      "tbz x26, #0, 16f\n"
-      "ld1 { v0.b }[2], [x25]\n"
+      "cbz x11, 18f\n"
+      "tbz x11, #1, 15f\n"
+      "ldr h0, [x10], #0x2\n"
+      "tbz x11, #0, 16f\n"
+      "ld1 { v0.b }[2], [x10]\n"
       "b 16f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 17f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "17:"  // Height 1: Multiply loop: unique 4: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       "18:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
       "tbnz %x[flags], #31, 19f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v1.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "neg v1.4s, v1.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "19:"  // Height 1: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "ldr q3, [x9, #0x30]\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
       "add v18.4s, v18.4s, v2.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v19.4s, v19.4s, v3.4s\n"
-      "add x9, x9, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 20f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
@@ -329,90 +317,90 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "srshl v17.4s, v17.4s, v0.4s\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 29f\n"
-      "tbz x11, #3, 24f\n"
-      "str d16, [x28], #0x8\n"
-      "tbz x11, #2, 22f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "tbz x11, #1, 21f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[14], [x28]\n"
+      "tbz x15, #3, 24f\n"
+      "str d16, [x14], #0x8\n"
+      "tbz x15, #2, 22f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "tbz x15, #1, 21f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[14], [x14]\n"
       "b 28f\n"
       "21:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[12], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[12], [x14]\n"
       "b 28f\n"
       "22:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 23f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[10], [x28]\n"
+      "tbz x15, #1, 23f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[10], [x14]\n"
       "b 28f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[8], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[8], [x14]\n"
       "b 28f\n"
       "24:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 26f\n"
-      "str s16, [x28], #0x4\n"
-      "tbz x11, #1, 25f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[6], [x28]\n"
+      "tbz x15, #2, 26f\n"
+      "str s16, [x14], #0x4\n"
+      "tbz x15, #1, 25f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[6], [x14]\n"
       "b 28f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[4], [x28]\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[4], [x14]\n"
       "b 28f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 27f\n"
-      "str h16, [x28], #0x2\n"
-      "tbz x11, #0, 28f\n"
-      "st1 { v16.b }[2], [x28]\n"
+      "tbz x15, #1, 27f\n"
+      "str h16, [x14], #0x2\n"
+      "tbz x15, #0, 28f\n"
+      "st1 { v16.b }[2], [x14]\n"
       "b 28f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
       "28:"  // Height 1: Partial direct writeback: Done
       "b 30f\n"
       "29:"  // Height 1: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
       "30:"  // Height 1: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 2b\n"
       "b 122f\n"
       "31:"  // Height 2
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "movi v15.16b, #0x1\n"
-      "mov x9, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "movi v15.16b, #0x1\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x14, %x[output_ptr]\n"
       "32:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -423,110 +411,98 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "33:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "34:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 35f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x27, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x12, 36f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
       "b 36f\n"
       "35:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
       "36:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 41f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 39f\n"
       "37:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
-      "ldr d6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
-      "ldr x23, [x10, #0x28]\n"
-      "add x22, x22, #0x10\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr d7, [x13, #0xa0]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr d8, [x13, #0xb0]\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
-      "ldr d6, [x10, #0x90]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d4, [x13, #0xe0]\n"
+      "mov v7.d[1], x26\n"
+      "ldr x24, [x13, #0xc8]\n"
+      "mov v8.d[1], x25\n"
+      "ldr x20, [x13, #0xd8]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "mov v6.d[1], x23\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "ldr d5, [x13, #0xf0]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xb8]\n"
-      "mov v7.d[1], x19\n"
-      "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v8.d[1], x23\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
+      "add x10, x10, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
-      "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
-      "ldr d5, [x10, #0xf0]\n"
-      "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8b7  // udot v23.4s, v5.16b, v1.4b[3]\n"
@@ -534,53 +510,53 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "38:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "bge 37b\n"
       "39:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
-      "ldr q7, [x10, #0x30]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
@@ -598,143 +574,143 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 6: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "41:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 48f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 48f\n"
+      "cmp x11, #0x4\n"
       "blt 44f\n"
       "42:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
       "tbnz %x[flags], #31, 43f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "43:"  // Height 2: Multiply loop: unique 7: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x6f81e116  // udot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       "bge 42b\n"
-      "cbz x26, 48f\n"
       "44:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 45f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "tbz x26, #0, 46f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
+      "cbz x11, 48f\n"
+      "tbz x11, #1, 45f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "tbz x11, #0, 46f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
       "b 46f\n"
       "45:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
       "46:"  // Height 2: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 47f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "47:"  // Height 2: Multiply loop: unique 8: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       "48:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 34b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "tbnz %x[flags], #31, 49f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "neg v2.4s, v2.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "49:"  // Height 2: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
-      "add v22.4s, v22.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
+      "add v22.4s, v22.4s, v2.4s\n"
       "add v23.4s, v23.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 50f\n"
       "and v4.16b, v16.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
+      "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
-      "and v7.16b, v19.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
+      "and v4.16b, v23.16b, v0.16b\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "50:"  // Height 2: no shift correction
       "srshl v16.4s, v16.4s, v0.4s\n"
@@ -745,13 +721,8 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "srshl v21.4s, v21.4s, v0.4s\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -760,6 +731,8 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "add v21.4s, v21.4s, v4.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -768,6 +741,8 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -780,87 +755,88 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 59f\n"
-      "tbz x11, #3, 54f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x11, #2, 52f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "tbz x11, #1, 51f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "tbz x15, #3, 54f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x15, #2, 52f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "tbz x15, #1, 51f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 58f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 58f\n"
       "52:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 53f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "tbz x15, #1, 53f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 58f\n"
       "53:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 58f\n"
       "54:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 56f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "tbz x11, #1, 55f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "tbz x15, #2, 56f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "tbz x15, #1, 55f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 58f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 58f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 57f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "tbz x11, #0, 58f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "tbz x15, #1, 57f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "tbz x15, #0, 58f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 58f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "58:"  // Height 2: Partial direct writeback: Done
       "b 60f\n"
       "59:"  // Height 2: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
       "60:"  // Height 2: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 32b\n"
       "b 122f\n"
       "61:"  // Height 3
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x9, %x[col_bias]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x14, %x[output_ptr]\n"
       "62:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -875,129 +851,117 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "63:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "64:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 65f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x27, 66f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x12, 66f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 66f\n"
       "65:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
-      "add x21, x22, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
+      "add x22, x23, x20\n"
       "66:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 71f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 69f\n"
       "67:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr d6, [x10, #0x20]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "add x25, x25, #0x10\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
-      "add x22, x22, #0x10\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "mov v8.d[1], x23\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr d10, [x10, #0x60]\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
-      "mov v9.d[1], x19\n"
+      "ldr d7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "mov v7.d[1], x26\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "mov v8.d[1], x25\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
-      "mov v4.d[1], x19\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr d6, [x10, #0x90]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
-      "mov v6.d[1], x23\n"
+      "ldr d4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr x19, [x10, #0xa8]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr d8, [x10, #0xb0]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
-      "ldr x23, [x10, #0xb8]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "mov v7.d[1], x19\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr d9, [x10, #0xc0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
-      "mov v8.d[1], x23\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
-      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6f82e91b  // udot v27.4s, v8.16b, v2.4b[2]\n"
-      "mov v10.d[1], x23\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
-      "ldr x19, [x10, #0xe8]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
-      "ldr d5, [x10, #0xf0]\n"
       ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
-      "ldr x24, [x10, #0xf8]\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6fa2e959  // udot v25.4s, v10.16b, v2.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x6fa2e89a  // udot v26.4s, v4.16b, v2.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
@@ -1008,65 +972,65 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "68:"  // Height 3: Multiply loop: unique 9: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
       "bge 67b\n"
       "69:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
@@ -1092,32 +1056,32 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "70:"  // Height 3: Multiply loop: unique 10: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "71:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 78f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 78f\n"
+      "cmp x11, #0x4\n"
       "blt 74f\n"
       "72:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
       "tbnz %x[flags], #31, 73f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "73:"  // Height 3: Multiply loop: unique 11: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f9  // udot v25.4s, v7.16b, v2.4b[0]\n"
@@ -1128,37 +1092,37 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
       "bge 72b\n"
-      "cbz x26, 78f\n"
       "74:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 75f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "tbz x26, #0, 76f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
+      "cbz x11, 78f\n"
+      "tbz x11, #1, 75f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "tbz x11, #0, 76f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
       "b 76f\n"
       "75:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
       "76:"  // Height 3: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 12: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f82e099  // udot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
@@ -1167,22 +1131,22 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
       "78:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 64b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbnz %x[flags], #31, 79f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "neg v3.4s, v3.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
@@ -1191,9 +1155,13 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "79:"  // Height 3: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
@@ -1203,73 +1171,69 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "add v25.4s, v25.4s, v13.4s\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
+      "add v23.4s, v23.4s, v3.4s\n"
       "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add v19.4s, v19.4s, v3.4s\n"
-      "add v23.4s, v23.4s, v3.4s\n"
       "add v27.4s, v27.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "sqrdmulh v24.4s, v24.4s, v4.4s\n"
       "sqrdmulh v25.4s, v25.4s, v4.4s\n"
       "sqrdmulh v26.4s, v26.4s, v4.4s\n"
       "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 80f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
-      "and v9.16b, v21.16b, v0.16b\n"
-      "and v10.16b, v22.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
-      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "and v9.16b, v21.16b, v0.16b\n"
+      "and v10.16b, v22.16b, v0.16b\n"
+      "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
       "and v6.16b, v25.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
@@ -1287,13 +1251,8 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "srshl v25.4s, v25.4s, v0.4s\n"
       "srshl v26.4s, v26.4s, v0.4s\n"
       "srshl v27.4s, v27.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -1304,6 +1263,10 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -1314,6 +1277,10 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -1324,120 +1291,117 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
+      "smax v26.4s, v26.4s, v5.4s\n"
+      "smax v27.4s, v27.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
+      "uzp1 v25.8h, v26.8h, v27.8h\n"
+      "cmp x15, #0x10\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
-      "smax v26.4s, v26.4s, v5.4s\n"
-      "smax v27.4s, v27.4s, v5.4s\n"
-      "uzp1 v25.8h, v26.8h, v27.8h\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 89f\n"
-      "tbz x11, #3, 84f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x11, #2, 82f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "tbz x11, #1, 81f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "tbz x15, #3, 84f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x15, #2, 82f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "tbz x15, #1, 81f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 88f\n"
       "81:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 88f\n"
       "82:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 83f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "tbz x15, #1, 83f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 88f\n"
       "83:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 88f\n"
       "84:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 86f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "tbz x11, #1, 85f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "tbz x15, #2, 86f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "tbz x15, #1, 85f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 88f\n"
       "85:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 88f\n"
       "86:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 87f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "tbz x11, #0, 88f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "tbz x15, #1, 87f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "tbz x15, #0, 88f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 88f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "88:"  // Height 3: Partial direct writeback: Done
       "b 90f\n"
       "89:"  // Height 3: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "90:"  // Height 3: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 62b\n"
       "b 122f\n"
       "91:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x16, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x9, %x[col_bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "movi v15.16b, #0x1\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x28, %x[output_ptr]\n"
-      "mov x19, #0x4\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x14, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "92:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1456,137 +1420,125 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "93:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x12, #0x0\n"
       "94:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w11, [x20, x12, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 95f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x27, 96f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x10, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x12, 96f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x10, x10, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 96f\n"
       "95:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x22, x25, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x10, %x[input_ptr]\n"
+      "add x23, x10, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "96:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x11, #0x10\n"
       "blt 101f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
+      "ldr q0, [x10, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q1, [x23, #0x0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
       "blt 99f\n"
       "97:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr d5, [x10, #0x10]\n"
+      "ldr x9, [x13, #0x78]\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr x24, [x10, #0x18]\n"
+      "ldr x28, [x13, #0x88]\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr d6, [x10, #0x20]\n"
+      "ldr x27, [x13, #0x98]\n"
       ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr x23, [x10, #0x28]\n"
-      "mov v5.d[1], x24\n"
-      "ldr d7, [x10, #0x30]\n"
-      "ldr x19, [x10, #0x38]\n"
-      "add x25, x25, #0x10\n"
+      "ldr d4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "mov v6.d[1], x23\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
+      "ldr x26, [x13, #0xa8]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "mov v7.d[1], x19\n"
+      "ldr x25, [x13, #0xb8]\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
-      "ldr x23, [x10, #0x48]\n"
+      "ldr d5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr d9, [x10, #0x50]\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr x19, [x10, #0x58]\n"
+      "ldr x24, [x13, #0xc8]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
-      "mov v8.d[1], x23\n"
+      "ldr x20, [x13, #0xd8]\n"
       ".inst 0x6f83e0de  // udot v30.4s, v6.16b, v3.4b[0]\n"
-      "ldr d10, [x10, #0x60]\n"
+      "ldr d6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "mov v9.d[1], x19\n"
+      "mov v6.d[1], x27\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr x23, [x10, #0x68]\n"
+      "ldr x9, [x13, #0xe8]\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr d4, [x10, #0x70]\n"
+      "ldr x28, [x13, #0xf8]\n"
       ".inst 0x6f83e0ff  // udot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr x19, [x10, #0x78]\n"
+      "ldr d7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "mov v7.d[1], x26\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr d5, [x10, #0x80]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
-      "mov v4.d[1], x19\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6fa3e11c  // udot v28.4s, v8.16b, v3.4b[1]\n"
-      "ldr x24, [x10, #0x88]\n"
+      "ldr d8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "ldr d6, [x10, #0x90]\n"
+      "mov v8.d[1], x25\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
-      "ldr x23, [x10, #0x98]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
-      "mov v5.d[1], x24\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6fa3e13d  // udot v29.4s, v9.16b, v3.4b[1]\n"
-      "ldr d7, [x10, #0xa0]\n"
+      "ldr d9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "mov v6.d[1], x23\n"
+      "mov v9.d[1], x24\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr x19, [x10, #0xa8]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x6fa3e15e  // udot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr x23, [x10, #0xb8]\n"
+      "ldr d10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "mov v7.d[1], x19\n"
+      "mov v10.d[1], x20\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
-      "ldr d9, [x10, #0xc0]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
-      "mov v8.d[1], x23\n"
       ".inst 0x6fa3e09f  // udot v31.4s, v4.16b, v3.4b[1]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      "ldr d4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr d10, [x10, #0xd0]\n"
+      "mov v4.d[1], x9\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
-      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x6f83e8bc  // udot v28.4s, v5.16b, v3.4b[2]\n"
-      "ldr d4, [x10, #0xe0]\n"
+      "ldr d5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "mov v10.d[1], x23\n"
+      "mov v5.d[1], x28\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr x19, [x10, #0xe8]\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
-      "ldr d5, [x10, #0xf0]\n"
       ".inst 0x6f83e8dd  // udot v29.4s, v6.16b, v3.4b[2]\n"
-      "ldr x24, [x10, #0xf8]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6f83e8fe  // udot v30.4s, v7.16b, v3.4b[2]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6f82e91b  // udot v27.4s, v8.16b, v2.4b[2]\n"
       ".inst 0x6f83e91f  // udot v31.4s, v8.16b, v3.4b[2]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
@@ -1611,77 +1563,77 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "98:"  // Height 4: Multiply loop: unique 13: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q0, [x10, #0x0]\n"
+      "sub x11, x11, #0x10\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x11, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x13, #0x0]\n"
+      "ldr q5, [x13, #0x10]\n"
+      "ldr q6, [x13, #0x20]\n"
+      "ldr q7, [x13, #0x30]\n"
+      "ldr q8, [x13, #0x40]\n"
+      "ldr q9, [x13, #0x50]\n"
+      "ldr q10, [x13, #0x60]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x10, #0x0]\n"
       "bge 97b\n"
       "99:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x10, #0x10]\n"
+      "sub x11, x11, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x20]\n"
+      "add x10, x10, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x10, #0x30]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
+      "ldr q4, [x13, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x50]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x10, #0x70]\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
-      "ldr q5, [x10, #0x80]\n"
+      "ldr q5, [x13, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f83e0de  // udot v30.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q6, [x13, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0ff  // udot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q7, [x13, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
       ".inst 0x6fa3e11c  // udot v28.4s, v8.16b, v3.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q8, [x13, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
       ".inst 0x6fa3e13d  // udot v29.4s, v9.16b, v3.4b[1]\n"
-      "ldr q9, [x10, #0xc0]\n"
+      "ldr q9, [x13, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
       ".inst 0x6fa3e15e  // udot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q10, [x13, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
       ".inst 0x6fa3e09f  // udot v31.4s, v4.16b, v3.4b[1]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q4, [x13, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
       ".inst 0x6f83e8bc  // udot v28.4s, v5.16b, v3.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q5, [x13, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "add x13, x13, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8dd  // udot v29.4s, v6.16b, v3.4b[2]\n"
@@ -1715,35 +1667,35 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "100:"  // Height 4: Multiply loop: unique 14: skip row sum
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x10, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "101:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 108f\n"
-      "cmp x26, #0x4\n"
+      "cbz x11, 108f\n"
+      "cmp x11, #0x4\n"
       "blt 104f\n"
       "102:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x10], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "tbnz %x[flags], #31, 103f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "103:"  // Height 4: Multiply loop: unique 15: skip row sum
-      "ldr q6, [x10, #0x0]\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q7, [x10, #0x10]\n"
-      "cmp x26, #0x4\n"
-      "ldr q8, [x10, #0x20]\n"
+      "ldr q6, [x13, #0x0]\n"
+      "sub x11, x11, #0x4\n"
+      "ldr q7, [x13, #0x10]\n"
+      "cmp x11, #0x4\n"
+      "ldr q8, [x13, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
+      "ldr q9, [x13, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f83e0dc  // udot v28.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
@@ -1758,24 +1710,24 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
       ".inst 0x6f83e13f  // udot v31.4s, v9.16b, v3.4b[0]\n"
       "bge 102b\n"
-      "cbz x26, 108f\n"
       "104:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 105f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "ldr h3, [x20], #0x2\n"
-      "tbz x26, #0, 106f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
-      "ld1 { v3.b }[2], [x20]\n"
+      "cbz x11, 108f\n"
+      "tbz x11, #1, 105f\n"
+      "ldr h0, [x10], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "ldr h3, [x21], #0x2\n"
+      "tbz x11, #0, 106f\n"
+      "ld1 { v0.b }[2], [x10]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
+      "ld1 { v3.b }[2], [x21]\n"
       "b 106f\n"
       "105:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
-      "ldr b3, [x20, #0x0]\n"
+      "ldr b0, [x10, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
+      "ldr b3, [x21, #0x0]\n"
       "106:"  // Height 4: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 107f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
@@ -1783,16 +1735,16 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "107:"  // Height 4: Multiply loop: unique 16: skip row sum
-      "ldr q10, [x10, #0x0]\n"
-      "ldr q4, [x10, #0x10]\n"
-      "ldr q5, [x10, #0x20]\n"
+      "ldr q10, [x13, #0x0]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
+      "ldr q4, [x13, #0x10]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
-      "add x10, x10, #0x40\n"
+      "ldr q5, [x13, #0x20]\n"
       ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
+      "ldr q6, [x13, #0x30]\n"
       ".inst 0x6f83e15c  // udot v28.4s, v10.16b, v3.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
+      "add x13, x13, #0x40\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x6f82e099  // udot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x6f83e09d  // udot v29.4s, v4.16b, v3.4b[0]\n"
@@ -1805,28 +1757,28 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0df  // udot v31.4s, v6.16b, v3.4b[0]\n"
       "108:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x12, x12, #0x1\n"
+      "cmp x12, x20\n"
       "bne 94b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "add x21, x28, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x14, x20\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x14, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "tbnz %x[flags], #31, 109f\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "neg v4.4s, v4.4s\n"
       "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "neg v4.4s, v4.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
       "mul v11.4s, v11.4s, v4.4s\n"
@@ -1834,9 +1786,13 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "109:"  // Height 4: skip row sum fixup
+      "ldr q0, [x16, #0x0]\n"
       "add v16.4s, v16.4s, v11.4s\n"
+      "ldr q1, [x16, #0x10]\n"
       "add v17.4s, v17.4s, v11.4s\n"
+      "ldr q2, [x16, #0x20]\n"
       "add v18.4s, v18.4s, v11.4s\n"
+      "ldr q3, [x16, #0x30]\n"
       "add v19.4s, v19.4s, v11.4s\n"
       "add v20.4s, v20.4s, v12.4s\n"
       "add v21.4s, v21.4s, v12.4s\n"
@@ -1850,39 +1806,34 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "add v29.4s, v29.4s, v14.4s\n"
       "add v30.4s, v30.4s, v14.4s\n"
       "add v31.4s, v31.4s, v14.4s\n"
-      "ldr q0, [x9, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
-      "ldr q1, [x9, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
-      "ldr q2, [x9, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "add v20.4s, v20.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
-      "add v21.4s, v21.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add v19.4s, v19.4s, v3.4s\n"
+      "add v20.4s, v20.4s, v0.4s\n"
+      "add v21.4s, v21.4s, v1.4s\n"
       "add v22.4s, v22.4s, v2.4s\n"
+      "add v23.4s, v23.4s, v3.4s\n"
       "add v24.4s, v24.4s, v0.4s\n"
       "add v25.4s, v25.4s, v1.4s\n"
       "add v26.4s, v26.4s, v2.4s\n"
+      "add v27.4s, v27.4s, v3.4s\n"
       "add v28.4s, v28.4s, v0.4s\n"
       "add v29.4s, v29.4s, v1.4s\n"
       "add v30.4s, v30.4s, v2.4s\n"
-      "ldr q3, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add v19.4s, v19.4s, v3.4s\n"
-      "add v23.4s, v23.4s, v3.4s\n"
-      "add v27.4s, v27.4s, v3.4s\n"
       "add v31.4s, v31.4s, v3.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v20.4s, v20.4s, v4.4s\n"
       "sqrdmulh v21.4s, v21.4s, v4.4s\n"
       "sqrdmulh v22.4s, v22.4s, v4.4s\n"
-      "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "sqrdmulh v24.4s, v24.4s, v4.4s\n"
       "sqrdmulh v25.4s, v25.4s, v4.4s\n"
@@ -1892,53 +1843,54 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "sqrdmulh v29.4s, v29.4s, v4.4s\n"
       "sqrdmulh v30.4s, v30.4s, v4.4s\n"
       "sqrdmulh v31.4s, v31.4s, v4.4s\n"
+      "add x16, x16, #0x40\n"
       "tbz %x[flags], #5, 110f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v16.4s, v16.4s, v4.4s\n"
+      "sqadd v17.4s, v17.4s, v5.4s\n"
       "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
+      "and v4.16b, v23.16b, v0.16b\n"
+      "and v5.16b, v24.16b, v0.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "sqadd v16.4s, v16.4s, v4.4s\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sqadd v17.4s, v17.4s, v5.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
-      "and v5.16b, v24.16b, v0.16b\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
       "and v9.16b, v28.16b, v0.16b\n"
       "and v10.16b, v29.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
+      "and v4.16b, v30.16b, v0.16b\n"
+      "and v5.16b, v31.16b, v0.16b\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
-      "and v4.16b, v30.16b, v0.16b\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
-      "and v5.16b, v31.16b, v0.16b\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "sqadd v28.4s, v28.4s, v9.4s\n"
       "sqadd v29.4s, v29.4s, v10.4s\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v30.4s, v30.4s, v4.4s\n"
       "sqadd v31.4s, v31.4s, v5.4s\n"
       "110:"  // Height 4: no shift correction
@@ -1958,13 +1910,8 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "srshl v29.4s, v29.4s, v0.4s\n"
       "srshl v30.4s, v30.4s, v0.4s\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "add x23, %x[qp], %[minval]\n"
-      "ld1r { v4.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      "ld1r { v5.4s }, [x23]\n"
-      "cmp x11, #0x10\n"
-      "ld1r { v6.4s }, [x22]\n"
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
@@ -1975,6 +1922,14 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
       "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
@@ -1985,6 +1940,14 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
       "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
@@ -1995,152 +1958,141 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "uzp1 v20.8h, v20.8h, v21.8h\n"
-      "uzp1 v21.8h, v22.8h, v23.8h\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "uzp1 v24.8h, v24.8h, v25.8h\n"
-      "uzp1 v16.16b, v16.16b, v17.16b\n"
-      "uzp1 v20.16b, v20.16b, v21.16b\n"
       "smax v26.4s, v26.4s, v5.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
       "smax v31.4s, v31.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v20.8h, v20.8h, v21.8h\n"
+      "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
       "uzp1 v28.8h, v28.8h, v29.8h\n"
       "uzp1 v29.8h, v30.8h, v31.8h\n"
+      "cmp x15, #0x10\n"
+      "uzp1 v16.16b, v16.16b, v17.16b\n"
+      "uzp1 v20.16b, v20.16b, v21.16b\n"
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 119f\n"
-      "tbz x11, #3, 114f\n"
-      "str d16, [x28], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x11, #2, 112f\n"
-      "st1 { v16.s }[2], [x28], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
-      "tbz x11, #1, 111f\n"
-      "st1 { v16.h }[6], [x28], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[14], [x28]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "tbz x15, #3, 114f\n"
+      "str d16, [x14], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x15, #2, 112f\n"
+      "st1 { v16.s }[2], [x14], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
+      "tbz x15, #1, 111f\n"
+      "st1 { v16.h }[6], [x14], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[14], [x14]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 118f\n"
       "111:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[12], [x28]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[12], [x14]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 118f\n"
       "112:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x11, #1, 113f\n"
-      "st1 { v16.h }[4], [x28], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[10], [x28]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "tbz x15, #1, 113f\n"
+      "st1 { v16.h }[4], [x14], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[10], [x14]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 118f\n"
       "113:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[8], [x28]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[8], [x14]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 118f\n"
       "114:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x11, #2, 116f\n"
-      "str s16, [x28], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
-      "tbz x11, #1, 115f\n"
-      "st1 { v16.h }[2], [x28], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[6], [x28]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "tbz x15, #2, 116f\n"
+      "str s16, [x14], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
+      "tbz x15, #1, 115f\n"
+      "st1 { v16.h }[2], [x14], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[6], [x14]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 118f\n"
       "115:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[4], [x28]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[4], [x14]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 118f\n"
       "116:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x11, #1, 117f\n"
-      "str h16, [x28], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
-      "tbz x11, #0, 118f\n"
-      "st1 { v16.b }[2], [x28]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "tbz x15, #1, 117f\n"
+      "str h16, [x14], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
+      "tbz x15, #0, 118f\n"
+      "st1 { v16.b }[2], [x14]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 118f\n"
       "117:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b16, [x28, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b16, [x14, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "118:"  // Height 4: Partial direct writeback: Done
       "b 120f\n"
       "119:"  // Height 4: Full writeback
-      "str q16, [x28, #0x0]\n"
-      "add x28, x28, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q16, [x14, #0x0]\n"
+      "add x14, x14, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "120:"  // Height 4: Writeback done
-      "subs x11, x11, #0x10\n"
+      "subs x15, x15, #0x10\n"
       "bgt 92b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 122f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "121:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "122:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp
index 4fc680c45b..31fbf88603 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,69 +85,69 @@ void a64_hybrid_u8qa_dot_4x16 (
       "cmp %x[M], #0x2\n"
       "bgt 61f\n"
       "beq 31f\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 11f\n"
-      "ldr q0, [x23, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 9f\n"
-      "7:"  // Height 1: Multiply loop: Main loop head
-      ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x20]\n"
       "ldr q7, [x28, #0x30]\n"
-      ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q8, [x28, #0x40]\n"
-      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x50]\n"
       "ldr q10, [x28, #0x60]\n"
-      ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "blt 9f\n"
+      "7:"  // Height 1: Multiply loop: Main loop head
+      ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x70]\n"
+      ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
-      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
+      ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
+      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
+      ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
-      ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
+      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
+      ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0xd0]\n"
-      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
+      ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       "ldr q4, [x28, #0xe0]\n"
+      ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
+      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
       "add x28, x28, #0x100\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
@@ -157,42 +157,42 @@ void a64_hybrid_u8qa_dot_4x16 (
       "tbnz %x[flags], #31, 8f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "ldr q0, [x23, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q0, [x24, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
-      "bge 7b\n"
-      "9:"  // Height 1: Multiply loop: Single iteration only
-      ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
-      ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x30]\n"
       "ldr q8, [x28, #0x40]\n"
-      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x50]\n"
-      ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "bge 7b\n"
+      "9:"  // Height 1: Multiply loop: Single iteration only
+      ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x70]\n"
-      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
+      ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
+      ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
-      ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
+      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0xa0]\n"
+      ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
-      ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
+      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
-      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       "ldr q10, [x28, #0xd0]\n"
+      ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       "ldr q4, [x28, #0xe0]\n"
+      ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
+      "ldr q5, [x28, #0xf0]\n"
+      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "ldr q5, [x28, #0xf0]\n"
+      "add x24, x24, #0x10\n"
       "add x28, x28, #0x100\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
@@ -201,83 +201,83 @@ void a64_hybrid_u8qa_dot_4x16 (
       "tbnz %x[flags], #31, 10f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 18f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 18f\n"
+      "cmp x25, #0x4\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
+      ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       "bge 12b\n"
-      "cbz x24, 18f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 15f\n"
-      "ldr h0, [x23], #0x2\n"
-      "tbz x24, #0, 16f\n"
-      "ld1 { v0.b }[2], [x23]\n"
+      "cbz x25, 18f\n"
+      "tbz x25, #1, 15f\n"
+      "ldr h0, [x24], #0x2\n"
+      "tbz x25, #0, 16f\n"
+      "ld1 { v0.b }[2], [x24]\n"
       "b 16f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 17f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "17:"  // Height 1: Multiply loop: unique 4: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       "18:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "tbnz %x[flags], #31, 19f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
-      "ld1r { v1.4s }, [x22]\n"
       "neg v1.4s, v1.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "19:"  // Height 1: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v16.4s, v16.4s, v0.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+      "add x10, x10, #0x40\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
@@ -289,100 +289,100 @@ void a64_hybrid_u8qa_dot_4x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "20:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "cmp x9, #0x10\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 29f\n"
       "tbz x9, #3, 24f\n"
-      "str d16, [x26], #0x8\n"
+      "str d16, [x27], #0x8\n"
       "tbz x9, #2, 22f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
       "tbz x9, #1, 21f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[14], [x26]\n"
+      "st1 { v16.b }[14], [x27]\n"
       "b 28f\n"
       "21:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[12], [x26]\n"
+      "st1 { v16.b }[12], [x27]\n"
       "b 28f\n"
       "22:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 23f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[10], [x26]\n"
+      "st1 { v16.b }[10], [x27]\n"
       "b 28f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[8], [x26]\n"
+      "st1 { v16.b }[8], [x27]\n"
       "b 28f\n"
       "24:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 26f\n"
-      "str s16, [x26], #0x4\n"
+      "str s16, [x27], #0x4\n"
       "tbz x9, #1, 25f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[6], [x26]\n"
+      "st1 { v16.b }[6], [x27]\n"
       "b 28f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[4], [x26]\n"
+      "st1 { v16.b }[4], [x27]\n"
       "b 28f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 27f\n"
-      "str h16, [x26], #0x2\n"
+      "str h16, [x27], #0x2\n"
       "tbz x9, #0, 28f\n"
-      "st1 { v16.b }[2], [x26]\n"
+      "st1 { v16.b }[2], [x27]\n"
       "b 28f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
       "28:"  // Height 1: Partial direct writeback: Done
       "b 30f\n"
       "29:"  // Height 1: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
       "30:"  // Height 1: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 2b\n"
       "b 122f\n"
       "31:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v15.16b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "32:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -393,57 +393,57 @@ void a64_hybrid_u8qa_dot_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "33:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "34:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 35f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 36f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 36f\n"
       "35:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "36:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 41f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 39f\n"
       "37:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
-      "ldr q4, [x28, #0x70]\n"
+      "ldr q6, [x28, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
+      "ldr q7, [x28, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x90]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
+      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
@@ -474,40 +474,40 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "38:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       "bge 37b\n"
       "39:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
-      "ldr q4, [x28, #0x70]\n"
+      "ldr q6, [x28, #0x90]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
+      "ldr q7, [x28, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      "ldr q6, [x28, #0x90]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
-      "ldr q7, [x28, #0xa0]\n"
-      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       "ldr q8, [x28, #0xb0]\n"
+      ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       "ldr q9, [x28, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
@@ -538,104 +538,104 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 6: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "41:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 48f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 48f\n"
+      "cmp x25, #0x4\n"
       "blt 44f\n"
       "42:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
       "tbnz %x[flags], #31, 43f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "43:"  // Height 2: Multiply loop: unique 7: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x6f81e116  // udot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       "bge 42b\n"
-      "cbz x24, 48f\n"
       "44:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 45f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "tbz x24, #0, 46f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
+      "cbz x25, 48f\n"
+      "tbz x25, #1, 45f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "tbz x25, #0, 46f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
       "b 46f\n"
       "45:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
       "46:"  // Height 2: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 47f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       "47:"  // Height 2: Multiply loop: unique 8: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
-      "add x28, x28, #0x40\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       "48:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 34b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "tbnz %x[flags], #31, 49f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v2.4s, v2.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "49:"  // Height 2: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v16.4s, v16.4s, v0.4s\n"
       "add v17.4s, v17.4s, v1.4s\n"
+      "add x10, x10, #0x40\n"
       "add v18.4s, v18.4s, v2.4s\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "add v20.4s, v20.4s, v0.4s\n"
@@ -653,154 +653,154 @@ void a64_hybrid_u8qa_dot_4x16 (
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
       "tbz %x[flags], #5, 50f\n"
       "and v4.16b, v16.16b, v0.16b\n"
-      "and v5.16b, v17.16b, v0.16b\n"
-      "and v6.16b, v18.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
-      "sqadd v17.4s, v17.4s, v5.4s\n"
-      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "and v5.16b, v17.16b, v0.16b\n"
+      "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
+      "and v10.16b, v22.16b, v0.16b\n"
+      "and v4.16b, v23.16b, v0.16b\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v17.4s, v17.4s, v5.4s\n"
+      "sqadd v18.4s, v18.4s, v6.4s\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
       "sqadd v21.4s, v21.4s, v9.4s\n"
-      "and v10.16b, v22.16b, v0.16b\n"
-      "and v4.16b, v23.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "50:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
+      "add v22.4s, v22.4s, v4.4s\n"
+      "add v23.4s, v23.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v20.4s, v20.4s, v6.4s\n"
       "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v20.4s, v20.4s, v5.4s\n"
       "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "add v22.4s, v22.4s, v4.4s\n"
-      "add v23.4s, v23.4s, v4.4s\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "uzp1 v16.16b, v16.16b, v17.16b\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
+      "uzp1 v16.16b, v16.16b, v17.16b\n"
       "uzp1 v20.16b, v20.16b, v21.16b\n"
       "bge 59f\n"
       "tbz x9, #3, 54f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
       "tbz x9, #2, 52f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
       "tbz x9, #1, 51f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
       "b 58f\n"
       "51:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
       "b 58f\n"
       "52:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 53f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
       "b 58f\n"
       "53:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
       "b 58f\n"
       "54:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 56f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
       "tbz x9, #1, 55f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
       "b 58f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
       "b 58f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 57f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
       "tbz x9, #0, 58f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
       "b 58f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
       "58:"  // Height 2: Partial direct writeback: Done
       "b 60f\n"
       "59:"  // Height 2: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
       "60:"  // Height 2: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 32b\n"
       "b 122f\n"
       "61:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "movi v15.16b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "62:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -815,56 +815,56 @@ void a64_hybrid_u8qa_dot_4x16 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "63:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "64:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 65f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 66f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 66f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 66f\n"
       "65:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "66:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 71f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 69f\n"
       "67:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "ldr q5, [x28, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
@@ -891,8 +891,8 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
@@ -918,37 +918,37 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "68:"  // Height 3: Multiply loop: unique 9: skip row sum
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q4, [x28, #0x0]\n"
       "bge 67b\n"
       "69:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "ldr q5, [x28, #0x80]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      "ldr q5, [x28, #0x80]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
@@ -975,8 +975,8 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
       "ldr q5, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
@@ -1002,33 +1002,33 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "70:"  // Height 3: Multiply loop: unique 10: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "71:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 78f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 78f\n"
+      "cmp x25, #0x4\n"
       "blt 74f\n"
       "72:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
       "tbnz %x[flags], #31, 73f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "73:"  // Height 3: Multiply loop: unique 11: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f9  // udot v25.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
@@ -1038,21 +1038,21 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
       "bge 72b\n"
-      "cbz x24, 78f\n"
       "74:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 75f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "tbz x24, #0, 76f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
+      "cbz x25, 78f\n"
+      "tbz x25, #1, 75f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "tbz x25, #0, 76f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
       "b 76f\n"
       "75:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
       "76:"  // Height 3: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
@@ -1060,16 +1060,16 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 12: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
-      ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x6f82e099  // udot v25.4s, v4.16b, v2.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x6f82e0ba  // udot v26.4s, v5.16b, v2.4b[0]\n"
@@ -1077,49 +1077,49 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
       "78:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 64b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "tbnz %x[flags], #31, 79f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v3.4s, v3.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "mul v11.4s, v11.4s, v3.4s\n"
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "79:"  // Height 3: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v16.4s, v16.4s, v0.4s\n"
@@ -1151,98 +1151,98 @@ void a64_hybrid_u8qa_dot_4x16 (
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
       "and v6.16b, v18.16b, v0.16b\n"
+      "and v7.16b, v19.16b, v0.16b\n"
+      "and v8.16b, v20.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "and v7.16b, v19.16b, v0.16b\n"
-      "and v8.16b, v20.16b, v0.16b\n"
-      "and v9.16b, v21.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "and v9.16b, v21.16b, v0.16b\n"
       "and v10.16b, v22.16b, v0.16b\n"
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v22.4s, v22.4s, v10.4s\n"
-      "sqadd v23.4s, v23.4s, v4.4s\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
+      "sqadd v22.4s, v22.4s, v10.4s\n"
+      "sqadd v23.4s, v23.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "80:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v20.8h, v20.8h, v21.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
@@ -1251,103 +1251,103 @@ void a64_hybrid_u8qa_dot_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 89f\n"
       "tbz x9, #3, 84f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x9, #2, 82f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x9, #1, 81f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 88f\n"
       "81:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 88f\n"
       "82:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 83f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 88f\n"
       "83:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 88f\n"
       "84:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 86f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x9, #1, 85f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 88f\n"
       "85:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 88f\n"
       "86:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 87f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x9, #0, 88f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 88f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "88:"  // Height 3: Partial direct writeback: Done
       "b 90f\n"
       "89:"  // Height 3: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "90:"  // Height 3: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 62b\n"
       "b 122f\n"
       "91:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x19, #0x4\n"
       "movi v15.16b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "92:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1366,59 +1366,59 @@ void a64_hybrid_u8qa_dot_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "93:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "94:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 95f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 96f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 96f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 96f\n"
       "95:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "96:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 101f\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
       "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
       "blt 99f\n"
       "97:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
@@ -1491,38 +1491,38 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "98:"  // Height 4: Multiply loop: unique 13: skip row sum
+      "ldr q0, [x24, #0x0]\n"
+      "ldr q1, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q3, [x21, #0x0]\n"
+      "ldr q4, [x28, #0x0]\n"
+      "ldr q5, [x28, #0x10]\n"
+      "ldr q6, [x28, #0x20]\n"
+      "ldr q7, [x28, #0x30]\n"
+      "ldr q8, [x28, #0x40]\n"
+      "ldr q9, [x28, #0x50]\n"
+      "ldr q10, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q0, [x23, #0x0]\n"
-      "ldr q1, [x22, #0x0]\n"
-      "ldr q2, [x21, #0x0]\n"
-      "ldr q3, [x20, #0x0]\n"
-      "ldr q4, [x28, #0x0]\n"
       "bge 97b\n"
       "99:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      "ldr q5, [x28, #0x10]\n"
-      "sub x24, x24, #0x10\n"
       ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      "ldr q6, [x28, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      "ldr q7, [x28, #0x30]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
-      "ldr q8, [x28, #0x40]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q4, [x28, #0x70]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
-      "ldr q9, [x28, #0x50]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
-      "ldr q10, [x28, #0x60]\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
-      "ldr q4, [x28, #0x70]\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
       "ldr q5, [x28, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
@@ -1595,19 +1595,19 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "100:"  // Height 4: Multiply loop: unique 14: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "101:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 108f\n"
-      "cmp x24, #0x4\n"
+      "cbz x25, 108f\n"
+      "cmp x25, #0x4\n"
       "blt 104f\n"
       "102:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x23], #0x4\n"
-      "ldr s1, [x22], #0x4\n"
-      "ldr s2, [x21], #0x4\n"
-      "ldr s3, [x20], #0x4\n"
+      "ldr s0, [x24], #0x4\n"
+      "ldr s1, [x23], #0x4\n"
+      "ldr s2, [x22], #0x4\n"
+      "ldr s3, [x21], #0x4\n"
       "tbnz %x[flags], #31, 103f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942c  // udot v12.4s, v1.16b, v15.16b\n"
@@ -1615,16 +1615,16 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "103:"  // Height 4: Multiply loop: unique 15: skip row sum
       "ldr q6, [x28, #0x0]\n"
-      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x28, #0x10]\n"
-      "sub x24, x24, #0x4\n"
-      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       "ldr q8, [x28, #0x20]\n"
-      "cmp x24, #0x4\n"
-      ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       "ldr q9, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0dc  // udot v28.4s, v6.16b, v3.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f9  // udot v25.4s, v7.16b, v2.4b[0]\n"
@@ -1638,24 +1638,24 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
       ".inst 0x6f83e13f  // udot v31.4s, v9.16b, v3.4b[0]\n"
       "bge 102b\n"
-      "cbz x24, 108f\n"
       "104:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x24, #1, 105f\n"
-      "ldr h0, [x23], #0x2\n"
-      "ldr h1, [x22], #0x2\n"
-      "ldr h2, [x21], #0x2\n"
-      "ldr h3, [x20], #0x2\n"
-      "tbz x24, #0, 106f\n"
-      "ld1 { v0.b }[2], [x23]\n"
-      "ld1 { v1.b }[2], [x22]\n"
-      "ld1 { v2.b }[2], [x21]\n"
-      "ld1 { v3.b }[2], [x20]\n"
+      "cbz x25, 108f\n"
+      "tbz x25, #1, 105f\n"
+      "ldr h0, [x24], #0x2\n"
+      "ldr h1, [x23], #0x2\n"
+      "ldr h2, [x22], #0x2\n"
+      "ldr h3, [x21], #0x2\n"
+      "tbz x25, #0, 106f\n"
+      "ld1 { v0.b }[2], [x24]\n"
+      "ld1 { v1.b }[2], [x23]\n"
+      "ld1 { v2.b }[2], [x22]\n"
+      "ld1 { v3.b }[2], [x21]\n"
       "b 106f\n"
       "105:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x23, #0x0]\n"
-      "ldr b1, [x22, #0x0]\n"
-      "ldr b2, [x21, #0x0]\n"
-      "ldr b3, [x20, #0x0]\n"
+      "ldr b0, [x24, #0x0]\n"
+      "ldr b1, [x23, #0x0]\n"
+      "ldr b2, [x22, #0x0]\n"
+      "ldr b3, [x21, #0x0]\n"
       "106:"  // Height 4: Multiply loop: Ragged operand read: Done
       "tbnz %x[flags], #31, 107f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
@@ -1664,16 +1664,16 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6e8f946e  // udot v14.4s, v3.16b, v15.16b\n"
       "107:"  // Height 4: Multiply loop: unique 16: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
       "ldr q5, [x28, #0x20]\n"
-      ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       "ldr q6, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x6f83e15c  // udot v28.4s, v10.16b, v3.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
+      "add x28, x28, #0x40\n"
       ".inst 0x6f82e099  // udot v25.4s, v4.16b, v2.4b[0]\n"
       ".inst 0x6f83e09d  // udot v29.4s, v4.16b, v3.4b[0]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
@@ -1685,27 +1685,27 @@ void a64_hybrid_u8qa_dot_4x16 (
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0df  // udot v31.4s, v6.16b, v3.4b[0]\n"
       "108:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 94b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "add x20, x21, x20\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "tbnz %x[flags], #31, 109f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
-      "addp v11.4s, v11.4s, v11.4s\n"
       "neg v4.4s, v4.4s\n"
+      "addp v11.4s, v11.4s, v11.4s\n"
       "addp v12.4s, v12.4s, v12.4s\n"
       "addp v13.4s, v13.4s, v13.4s\n"
       "addp v14.4s, v14.4s, v14.4s\n"
@@ -1714,25 +1714,25 @@ void a64_hybrid_u8qa_dot_4x16 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "109:"  // Height 4: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v16.4s, v16.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v20.4s, v20.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v21.4s, v21.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v22.4s, v22.4s, v12.4s\n"
       "add v23.4s, v23.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v28.4s, v28.4s, v14.4s\n"
@@ -1775,126 +1775,126 @@ void a64_hybrid_u8qa_dot_4x16 (
       "tbz %x[flags], #5, 110f\n"
       "and v4.16b, v16.16b, v0.16b\n"
       "and v5.16b, v17.16b, v0.16b\n"
-      "and v6.16b, v18.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v16.4s, v16.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
-      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "and v6.16b, v18.16b, v0.16b\n"
       "and v7.16b, v19.16b, v0.16b\n"
       "and v8.16b, v20.16b, v0.16b\n"
       "and v9.16b, v21.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sqadd v19.4s, v19.4s, v7.4s\n"
-      "sqadd v20.4s, v20.4s, v8.4s\n"
-      "sqadd v21.4s, v21.4s, v9.4s\n"
       "and v10.16b, v22.16b, v0.16b\n"
       "and v4.16b, v23.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v18.4s, v18.4s, v6.4s\n"
+      "sqadd v19.4s, v19.4s, v7.4s\n"
+      "sqadd v20.4s, v20.4s, v8.4s\n"
+      "sqadd v21.4s, v21.4s, v9.4s\n"
       "sqadd v22.4s, v22.4s, v10.4s\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sqadd v25.4s, v25.4s, v6.4s\n"
-      "sqadd v26.4s, v26.4s, v7.4s\n"
-      "sqadd v27.4s, v27.4s, v8.4s\n"
       "and v9.16b, v28.16b, v0.16b\n"
       "and v10.16b, v29.16b, v0.16b\n"
       "and v4.16b, v30.16b, v0.16b\n"
+      "and v5.16b, v31.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v25.4s, v25.4s, v6.4s\n"
+      "sqadd v26.4s, v26.4s, v7.4s\n"
+      "sqadd v27.4s, v27.4s, v8.4s\n"
       "sqadd v28.4s, v28.4s, v9.4s\n"
       "sqadd v29.4s, v29.4s, v10.4s\n"
       "sqadd v30.4s, v30.4s, v4.4s\n"
-      "and v5.16b, v31.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v5.4s\n"
       "110:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v22.4s, v22.4s, v0.4s\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
+      "srshl v28.4s, v28.4s, v0.4s\n"
+      "srshl v29.4s, v29.4s, v0.4s\n"
+      "srshl v30.4s, v30.4s, v0.4s\n"
+      "srshl v31.4s, v31.4s, v0.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
-      "srshl v22.4s, v22.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v31.4s, v31.4s, v4.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "srshl v28.4s, v28.4s, v0.4s\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
       "smax v27.4s, v27.4s, v5.4s\n"
-      "srshl v29.4s, v29.4s, v0.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
-      "srshl v30.4s, v30.4s, v0.4s\n"
-      "srshl v31.4s, v31.4s, v0.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "add v31.4s, v31.4s, v4.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
-      "uzp1 v17.8h, v18.8h, v19.8h\n"
       "smax v31.4s, v31.4s, v5.4s\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
+      "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v20.8h, v20.8h, v21.8h\n"
       "uzp1 v21.8h, v22.8h, v23.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
@@ -1907,120 +1907,120 @@ void a64_hybrid_u8qa_dot_4x16 (
       "uzp1 v28.16b, v28.16b, v29.16b\n"
       "bge 119f\n"
       "tbz x9, #3, 114f\n"
-      "str d16, [x26], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
+      "str d16, [x27], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
       "tbz x9, #2, 112f\n"
-      "st1 { v16.s }[2], [x26], #0x4\n"
-      "st1 { v20.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
-      "st1 { v28.s }[2], [x19], #0x4\n"
+      "st1 { v16.s }[2], [x27], #0x4\n"
+      "st1 { v20.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
+      "st1 { v28.s }[2], [x20], #0x4\n"
       "tbz x9, #1, 111f\n"
-      "st1 { v16.h }[6], [x26], #0x2\n"
-      "st1 { v20.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
-      "st1 { v28.h }[6], [x19], #0x2\n"
+      "st1 { v16.h }[6], [x27], #0x2\n"
+      "st1 { v20.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
+      "st1 { v28.h }[6], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[14], [x26]\n"
-      "st1 { v20.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
-      "st1 { v28.b }[14], [x19]\n"
+      "st1 { v16.b }[14], [x27]\n"
+      "st1 { v20.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
+      "st1 { v28.b }[14], [x20]\n"
       "b 118f\n"
       "111:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[12], [x26]\n"
-      "st1 { v20.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
-      "st1 { v28.b }[12], [x19]\n"
+      "st1 { v16.b }[12], [x27]\n"
+      "st1 { v20.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
+      "st1 { v28.b }[12], [x20]\n"
       "b 118f\n"
       "112:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 113f\n"
-      "st1 { v16.h }[4], [x26], #0x2\n"
-      "st1 { v20.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
-      "st1 { v28.h }[4], [x19], #0x2\n"
+      "st1 { v16.h }[4], [x27], #0x2\n"
+      "st1 { v20.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
+      "st1 { v28.h }[4], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[10], [x26]\n"
-      "st1 { v20.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
-      "st1 { v28.b }[10], [x19]\n"
+      "st1 { v16.b }[10], [x27]\n"
+      "st1 { v20.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
+      "st1 { v28.b }[10], [x20]\n"
       "b 118f\n"
       "113:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[8], [x26]\n"
-      "st1 { v20.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
-      "st1 { v28.b }[8], [x19]\n"
+      "st1 { v16.b }[8], [x27]\n"
+      "st1 { v20.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
+      "st1 { v28.b }[8], [x20]\n"
       "b 118f\n"
       "114:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 116f\n"
-      "str s16, [x26], #0x4\n"
-      "str s20, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
-      "str s28, [x19], #0x4\n"
+      "str s16, [x27], #0x4\n"
+      "str s20, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
+      "str s28, [x20], #0x4\n"
       "tbz x9, #1, 115f\n"
-      "st1 { v16.h }[2], [x26], #0x2\n"
-      "st1 { v20.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
-      "st1 { v28.h }[2], [x19], #0x2\n"
+      "st1 { v16.h }[2], [x27], #0x2\n"
+      "st1 { v20.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
+      "st1 { v28.h }[2], [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[6], [x26]\n"
-      "st1 { v20.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
-      "st1 { v28.b }[6], [x19]\n"
+      "st1 { v16.b }[6], [x27]\n"
+      "st1 { v20.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
+      "st1 { v28.b }[6], [x20]\n"
       "b 118f\n"
       "115:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[4], [x26]\n"
-      "st1 { v20.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
-      "st1 { v28.b }[4], [x19]\n"
+      "st1 { v16.b }[4], [x27]\n"
+      "st1 { v20.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
+      "st1 { v28.b }[4], [x20]\n"
       "b 118f\n"
       "116:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 117f\n"
-      "str h16, [x26], #0x2\n"
-      "str h20, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
-      "str h28, [x19], #0x2\n"
+      "str h16, [x27], #0x2\n"
+      "str h20, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
+      "str h28, [x20], #0x2\n"
       "tbz x9, #0, 118f\n"
-      "st1 { v16.b }[2], [x26]\n"
-      "st1 { v20.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
-      "st1 { v28.b }[2], [x19]\n"
+      "st1 { v16.b }[2], [x27]\n"
+      "st1 { v20.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
+      "st1 { v28.b }[2], [x20]\n"
       "b 118f\n"
       "117:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b16, [x26, #0x0]\n"
-      "str b20, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
-      "str b28, [x19, #0x0]\n"
+      "str b16, [x27, #0x0]\n"
+      "str b20, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
+      "str b28, [x20, #0x0]\n"
       "118:"  // Height 4: Partial direct writeback: Done
       "b 120f\n"
       "119:"  // Height 4: Full writeback
-      "str q16, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q20, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q28, [x19, #0x0]\n"
+      "str q16, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q20, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q28, [x20, #0x0]\n"
       "120:"  // Height 4: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 92b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 122f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 121f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "121:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "122:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp
index daeb986529..f808cb199d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -85,13 +85,13 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "cmp %x[M], #0x2\n"
       "bgt 65f\n"
       "beq 33f\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v15.16b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -102,27 +102,27 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 11f\n"
-      "ldr q1, [x23, #0x0]\n"
+      "ldr q1, [x24, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "cmp x25, #0x20\n"
       "ldr q6, [x28, #0x10]\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
@@ -132,10 +132,9 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "blt 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
@@ -152,9 +151,10 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "ldr q5, [x28, #0xe0]\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x6e87a434  // ummla v20.4s, v1.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e88a431  // ummla v17.4s, v1.16b, v8.16b\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x6e89a435  // ummla v21.4s, v1.16b, v9.16b\n"
       ".inst 0x6e8aa432  // ummla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x6e84a436  // ummla v22.4s, v1.16b, v4.16b\n"
@@ -164,25 +164,23 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "ldr q1, [x23, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q1, [x24, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q6, [x28, #0x10]\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
       "ldr q9, [x28, #0x40]\n"
       "ldr q10, [x28, #0x50]\n"
       "ldr q4, [x28, #0x60]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "bge 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "sub x24, x24, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
@@ -199,9 +197,11 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "ldr q5, [x28, #0xe0]\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
+      "sub x25, x25, #0x10\n"
       ".inst 0x6e87a434  // ummla v20.4s, v1.16b, v7.16b\n"
       ".inst 0x6e88a431  // ummla v17.4s, v1.16b, v8.16b\n"
+      "add x24, x24, #0x10\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x6e89a435  // ummla v21.4s, v1.16b, v9.16b\n"
       ".inst 0x6e8aa432  // ummla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x6e84a436  // ummla v22.4s, v1.16b, v4.16b\n"
@@ -211,120 +211,118 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       "10:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x24, 20f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 20f\n"
+      "cmp x25, #0x8\n"
       "blt 14f\n"
       "12:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x23], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 13f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "13:"  // Height 1: Multiply loop: unique 3: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
-      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
+      "sub x25, x25, #0x8\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x30]\n"
+      "cmp x25, #0x8\n"
+      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x6e86a416  // ummla v22.4s, v0.16b, v6.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e87a413  // ummla v19.4s, v0.16b, v7.16b\n"
       ".inst 0x6e88a417  // ummla v23.4s, v0.16b, v8.16b\n"
+      "add x28, x28, #0x80\n"
       "bge 12b\n"
-      "cbz x24, 20f\n"
       "14:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 16f\n"
-      "ldr s1, [x23], #0x4\n"
-      "tbz x24, #1, 15f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[6], [x23]\n"
+      "cbz x25, 20f\n"
+      "tbz x25, #2, 16f\n"
+      "ldr s1, [x24], #0x4\n"
+      "tbz x25, #1, 15f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[6], [x24]\n"
       "b 18f\n"
       "15:"  // Height 1: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[4], [x23]\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[4], [x24]\n"
       "b 18f\n"
       "16:"  // Height 1: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 17f\n"
-      "ldr h1, [x23], #0x2\n"
-      "tbz x24, #0, 18f\n"
-      "ld1 { v1.b }[2], [x23]\n"
+      "tbz x25, #1, 17f\n"
+      "ldr h1, [x24], #0x2\n"
+      "tbz x25, #0, 18f\n"
+      "ld1 { v1.b }[2], [x24]\n"
       "b 18f\n"
       "17:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
       "18:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 19f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "19:"  // Height 1: Multiply loop: unique 4: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x6e84a414  // ummla v20.4s, v0.16b, v4.16b\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x6e85a411  // ummla v17.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
       "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
-      ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x60]\n"
-      "ldr q10, [x28, #0x70]\n"
       ".inst 0x6e87a412  // ummla v18.4s, v0.16b, v7.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e88a416  // ummla v22.4s, v0.16b, v8.16b\n"
+      "ldr q9, [x28, #0x60]\n"
+      "ldr q10, [x28, #0x70]\n"
       ".inst 0x6e89a413  // ummla v19.4s, v0.16b, v9.16b\n"
       ".inst 0x6e8aa417  // ummla v23.4s, v0.16b, v10.16b\n"
+      "add x28, x28, #0x80\n"
       "20:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "uzp1 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
       "uzp1 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "mov v23.16b, v16.16b\n"
       "tbnz %x[flags], #31, 21f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v1.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v1.4s }, [x22]\n"
-      "dup v11.4s, v11.s[0]\n"
       "neg v1.4s, v1.4s\n"
+      "dup v11.4s, v11.s[0]\n"
       "mul v11.4s, v11.4s, v1.4s\n"
       "21:"  // Height 1: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v23.4s, v23.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v17.4s, v17.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v18.4s, v18.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v19.4s, v19.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v23.4s, v23.4s, v0.4s\n"
-      "ld1r { v0.4s }, [x23]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v1.4s\n"
       "add v18.4s, v18.4s, v2.4s\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1r { v0.4s }, [x23]\n"
       "add v19.4s, v19.4s, v3.4s\n"
       "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+      "add x10, x10, #0x40\n"
       "sqrdmulh v17.4s, v17.4s, v4.4s\n"
       "sqrdmulh v18.4s, v18.4s, v4.4s\n"
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
@@ -336,100 +334,100 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
       "sqadd v17.4s, v17.4s, v5.4s\n"
       "sqadd v18.4s, v18.4s, v6.4s\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
       "sqadd v19.4s, v19.4s, v7.4s\n"
       "22:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v18.4s, v18.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v19.4s, v19.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
+      "cmp x9, #0x10\n"
       "smin v23.4s, v23.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v23.8h, v23.8h, v17.8h\n"
       "smax v19.4s, v19.4s, v5.4s\n"
+      "uzp1 v23.8h, v23.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v23.16b, v23.16b, v17.16b\n"
       "bge 31f\n"
       "tbz x9, #3, 26f\n"
-      "str d23, [x26], #0x8\n"
+      "str d23, [x27], #0x8\n"
       "tbz x9, #2, 24f\n"
-      "st1 { v23.s }[2], [x26], #0x4\n"
+      "st1 { v23.s }[2], [x27], #0x4\n"
       "tbz x9, #1, 23f\n"
-      "st1 { v23.h }[6], [x26], #0x2\n"
+      "st1 { v23.h }[6], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[14], [x26]\n"
+      "st1 { v23.b }[14], [x27]\n"
       "b 30f\n"
       "23:"  // Height 1: Partial direct writeback: partial_1_12
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[12], [x26]\n"
+      "st1 { v23.b }[12], [x27]\n"
       "b 30f\n"
       "24:"  // Height 1: Partial direct writeback: partial_2_8
       "tbz x9, #1, 25f\n"
-      "st1 { v23.h }[4], [x26], #0x2\n"
+      "st1 { v23.h }[4], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[10], [x26]\n"
+      "st1 { v23.b }[10], [x27]\n"
       "b 30f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_8
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[8], [x26]\n"
+      "st1 { v23.b }[8], [x27]\n"
       "b 30f\n"
       "26:"  // Height 1: Partial direct writeback: partial_4_0
       "tbz x9, #2, 28f\n"
-      "str s23, [x26], #0x4\n"
+      "str s23, [x27], #0x4\n"
       "tbz x9, #1, 27f\n"
-      "st1 { v23.h }[2], [x26], #0x2\n"
+      "st1 { v23.h }[2], [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[6], [x26]\n"
+      "st1 { v23.b }[6], [x27]\n"
       "b 30f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_4
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[4], [x26]\n"
+      "st1 { v23.b }[4], [x27]\n"
       "b 30f\n"
       "28:"  // Height 1: Partial direct writeback: partial_2_0
       "tbz x9, #1, 29f\n"
-      "str h23, [x26], #0x2\n"
+      "str h23, [x27], #0x2\n"
       "tbz x9, #0, 30f\n"
-      "st1 { v23.b }[2], [x26]\n"
+      "st1 { v23.b }[2], [x27]\n"
       "b 30f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_0
-      "str b23, [x26, #0x0]\n"
+      "str b23, [x27, #0x0]\n"
       "30:"  // Height 1: Partial direct writeback: Done
       "b 32f\n"
       "31:"  // Height 1: Full writeback
-      "str q23, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
+      "str q23, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
       "32:"  // Height 1: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 2b\n"
       "b 130f\n"
       "33:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v15.16b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "34:"  // Height 2: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -440,64 +438,64 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "35:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "36:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 37f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 38f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 38f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 38f\n"
       "37:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "38:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 43f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 41f\n"
-      "39:"  // Height 2: Multiply loop: Main loop head
-      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q7, [x28, #0x20]\n"
       "ldr q8, [x28, #0x30]\n"
-      ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q9, [x28, #0x40]\n"
-      ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       "ldr q10, [x28, #0x50]\n"
-      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       "ldr q4, [x28, #0x60]\n"
-      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
+      "blt 41f\n"
+      "39:"  // Height 2: Multiply loop: Main loop head
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
-      ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
-      ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
-      ".inst 0x6e85a417  // ummla v23.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
+      ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
+      ".inst 0x6e85a417  // ummla v23.4s, v0.16b, v5.16b\n"
+      "ldr q5, [x28, #0xe0]\n"
+      ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
+      "ldr q6, [x28, #0xf0]\n"
       ".inst 0x6e87a434  // ummla v20.4s, v1.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e88a431  // ummla v17.4s, v1.16b, v8.16b\n"
-      "ldr q5, [x28, #0xe0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e89a435  // ummla v21.4s, v1.16b, v9.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
       "add x28, x28, #0x100\n"
       ".inst 0x6e8aa432  // ummla v18.4s, v1.16b, v10.16b\n"
       ".inst 0x6e84a436  // ummla v22.4s, v1.16b, v4.16b\n"
@@ -507,49 +505,49 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       "40:"  // Height 2: Multiply loop: unique 5: skip row sum
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "bge 39b\n"
-      "41:"  // Height 2: Multiply loop: Single iteration only
-      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
       "ldr q5, [x28, #0x0]\n"
-      "sub x24, x24, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q7, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q8, [x28, #0x30]\n"
       "ldr q9, [x28, #0x40]\n"
-      ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       "ldr q10, [x28, #0x50]\n"
       "ldr q4, [x28, #0x60]\n"
-      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
-      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "bge 39b\n"
+      "41:"  // Height 2: Multiply loop: Single iteration only
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0x70]\n"
+      ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
       "ldr q6, [x28, #0x80]\n"
-      ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       "ldr q7, [x28, #0x90]\n"
-      ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
-      ".inst 0x6e85a417  // ummla v23.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
+      ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
-      ".inst 0x6e87a434  // ummla v20.4s, v1.16b, v7.16b\n"
+      ".inst 0x6e85a417  // ummla v23.4s, v0.16b, v5.16b\n"
       "ldr q5, [x28, #0xe0]\n"
+      ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
+      "ldr q6, [x28, #0xf0]\n"
+      "sub x25, x25, #0x10\n"
+      ".inst 0x6e87a434  // ummla v20.4s, v1.16b, v7.16b\n"
       ".inst 0x6e88a431  // ummla v17.4s, v1.16b, v8.16b\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e89a435  // ummla v21.4s, v1.16b, v9.16b\n"
-      "ldr q6, [x28, #0xf0]\n"
-      "add x28, x28, #0x100\n"
       ".inst 0x6e8aa432  // ummla v18.4s, v1.16b, v10.16b\n"
+      "add x28, x28, #0x100\n"
       ".inst 0x6e84a436  // ummla v22.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a433  // ummla v19.4s, v1.16b, v5.16b\n"
       ".inst 0x6e86a437  // ummla v23.4s, v1.16b, v6.16b\n"
@@ -557,136 +555,136 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       "42:"  // Height 2: Multiply loop: unique 6: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       "43:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x24, 52f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 52f\n"
+      "cmp x25, #0x8\n"
       "blt 46f\n"
       "44:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 45f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "45:"  // Height 2: Multiply loop: unique 7: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
-      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
+      "sub x25, x25, #0x8\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x30]\n"
+      "cmp x25, #0x8\n"
+      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x6e86a416  // ummla v22.4s, v0.16b, v6.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e87a413  // ummla v19.4s, v0.16b, v7.16b\n"
       ".inst 0x6e88a417  // ummla v23.4s, v0.16b, v8.16b\n"
+      "add x28, x28, #0x80\n"
       "bge 44b\n"
-      "cbz x24, 52f\n"
       "46:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 48f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "tbz x24, #1, 47f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
+      "cbz x25, 52f\n"
+      "tbz x25, #2, 48f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "tbz x25, #1, 47f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
       "b 50f\n"
       "47:"  // Height 2: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
       "b 50f\n"
       "48:"  // Height 2: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 49f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "tbz x24, #0, 50f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
+      "tbz x25, #1, 49f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "tbz x25, #0, 50f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
       "b 50f\n"
       "49:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
       "50:"  // Height 2: Multiply loop: Ragged operand read: Done
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "tbnz %x[flags], #31, 51f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       "51:"  // Height 2: Multiply loop: unique 8: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
-      "ldr q5, [x28, #0x20]\n"
+      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x6e84a414  // ummla v20.4s, v0.16b, v4.16b\n"
+      "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x6e85a411  // ummla v17.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
       "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
-      ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x60]\n"
-      "ldr q10, [x28, #0x70]\n"
       ".inst 0x6e87a412  // ummla v18.4s, v0.16b, v7.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e88a416  // ummla v22.4s, v0.16b, v8.16b\n"
+      "ldr q9, [x28, #0x60]\n"
+      "ldr q10, [x28, #0x70]\n"
       ".inst 0x6e89a413  // ummla v19.4s, v0.16b, v9.16b\n"
       ".inst 0x6e8aa417  // ummla v23.4s, v0.16b, v10.16b\n"
+      "add x28, x28, #0x80\n"
       "52:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 36b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "mov v23.16b, v4.16b\n"
       "tbnz %x[flags], #31, 53f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v2.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v2.4s }, [x22]\n"
+      "neg v2.4s, v2.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v2.4s, v2.4s\n"
       "mul v11.4s, v11.4s, v2.4s\n"
       "mul v12.4s, v12.4s, v2.4s\n"
       "53:"  // Height 2: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v23.4s, v23.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v23.4s, v23.4s, v0.4s\n"
       "add v20.4s, v20.4s, v1.4s\n"
+      "add x10, x10, #0x40\n"
       "add v21.4s, v21.4s, v2.4s\n"
       "add v22.4s, v22.4s, v3.4s\n"
       "add v16.4s, v16.4s, v0.4s\n"
@@ -704,154 +702,154 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "sqrdmulh v19.4s, v19.4s, v4.4s\n"
       "tbz %x[flags], #5, 54f\n"
       "and v4.16b, v23.16b, v0.16b\n"
-      "and v5.16b, v20.16b, v0.16b\n"
-      "and v6.16b, v21.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v23.4s, v23.4s, v4.4s\n"
-      "sqadd v20.4s, v20.4s, v5.4s\n"
-      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "and v5.16b, v20.16b, v0.16b\n"
+      "and v6.16b, v21.16b, v0.16b\n"
       "and v7.16b, v22.16b, v0.16b\n"
       "and v8.16b, v16.16b, v0.16b\n"
       "and v9.16b, v17.16b, v0.16b\n"
+      "and v10.16b, v18.16b, v0.16b\n"
+      "and v4.16b, v19.16b, v0.16b\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sqadd v20.4s, v20.4s, v5.4s\n"
+      "sqadd v21.4s, v21.4s, v6.4s\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "sqadd v16.4s, v16.4s, v8.4s\n"
       "sqadd v17.4s, v17.4s, v9.4s\n"
-      "and v10.16b, v18.16b, v0.16b\n"
-      "and v4.16b, v19.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
       "sqadd v18.4s, v18.4s, v10.4s\n"
       "sqadd v19.4s, v19.4s, v4.4s\n"
       "54:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v23.4s, v23.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
       "add v23.4s, v23.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v23.4s, v23.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v23.4s, v23.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
+      "add v18.4s, v18.4s, v4.4s\n"
+      "add v19.4s, v19.4s, v4.4s\n"
+      "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
       "smin v22.4s, v22.4s, v6.4s\n"
       "smin v16.4s, v16.4s, v6.4s\n"
       "smin v17.4s, v17.4s, v6.4s\n"
+      "smin v18.4s, v18.4s, v6.4s\n"
+      "smin v19.4s, v19.4s, v6.4s\n"
+      "smax v23.4s, v23.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
       "smax v22.4s, v22.4s, v5.4s\n"
       "smax v16.4s, v16.4s, v5.4s\n"
       "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
+      "smax v18.4s, v18.4s, v5.4s\n"
+      "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v23.8h, v23.8h, v20.8h\n"
       "uzp1 v20.8h, v21.8h, v22.8h\n"
-      "add v18.4s, v18.4s, v4.4s\n"
-      "add v19.4s, v19.4s, v4.4s\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
-      "smin v18.4s, v18.4s, v6.4s\n"
-      "smin v19.4s, v19.4s, v6.4s\n"
-      "uzp1 v23.16b, v23.16b, v20.16b\n"
-      "smax v18.4s, v18.4s, v5.4s\n"
-      "smax v19.4s, v19.4s, v5.4s\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
+      "uzp1 v23.16b, v23.16b, v20.16b\n"
       "uzp1 v16.16b, v16.16b, v17.16b\n"
       "bge 63f\n"
       "tbz x9, #3, 58f\n"
-      "str d23, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
+      "str d23, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
       "tbz x9, #2, 56f\n"
-      "st1 { v23.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
+      "st1 { v23.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
       "tbz x9, #1, 55f\n"
-      "st1 { v23.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
+      "st1 { v23.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
+      "st1 { v23.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
       "b 62f\n"
       "55:"  // Height 2: Partial direct writeback: partial_1_12
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
+      "st1 { v23.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
       "b 62f\n"
       "56:"  // Height 2: Partial direct writeback: partial_2_8
       "tbz x9, #1, 57f\n"
-      "st1 { v23.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
+      "st1 { v23.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
+      "st1 { v23.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
       "b 62f\n"
       "57:"  // Height 2: Partial direct writeback: partial_1_8
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
+      "st1 { v23.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
       "b 62f\n"
       "58:"  // Height 2: Partial direct writeback: partial_4_0
       "tbz x9, #2, 60f\n"
-      "str s23, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
+      "str s23, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
       "tbz x9, #1, 59f\n"
-      "st1 { v23.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
+      "st1 { v23.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
+      "st1 { v23.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
       "b 62f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_4
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
+      "st1 { v23.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
       "b 62f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_0
       "tbz x9, #1, 61f\n"
-      "str h23, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
+      "str h23, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
       "tbz x9, #0, 62f\n"
-      "st1 { v23.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
+      "st1 { v23.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
       "b 62f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_0
-      "str b23, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
+      "str b23, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
       "62:"  // Height 2: Partial direct writeback: Done
       "b 64f\n"
       "63:"  // Height 2: Full writeback
-      "str q23, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
+      "str q23, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
       "64:"  // Height 2: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 34b\n"
       "b 130f\n"
       "65:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "movi v15.16b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "66:"  // Height 3: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -870,65 +868,65 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "67:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "68:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 69f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 70f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 70f\n"
       "69:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "70:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 75f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
-      "blt 73f\n"
-      "71:"  // Height 3: Multiply loop: Main loop head
-      "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
       "ldr q5, [x28, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
       "ldr q6, [x28, #0x10]\n"
-      "add x22, x22, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q7, [x28, #0x20]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
       "ldr q8, [x28, #0x30]\n"
-      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
       "ldr q9, [x28, #0x40]\n"
       "ldr q10, [x28, #0x50]\n"
+      "blt 73f\n"
+      "71:"  // Height 3: Multiply loop: Main loop head
+      "trn1 v0.2d, v1.2d, v2.2d\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
+      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
+      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
-      ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q4, [x28, #0x60]\n"
+      ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
-      "ldr q6, [x28, #0x80]\n"
-      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45d  // ummla v29.4s, v2.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x6e89a45a  // ummla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45e  // ummla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
@@ -962,49 +960,49 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f946d  // udot v13.4s, v3.16b, v15.16b\n"
       "72:"  // Height 3: Multiply loop: unique 9: skip row sum
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
       "bge 71b\n"
       "73:"  // Height 3: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "sub x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x28, #0x20]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
-      "add x21, x21, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      "ldr q9, [x28, #0x40]\n"
-      "ldr q10, [x28, #0x50]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
-      ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       "ldr q4, [x28, #0x60]\n"
+      ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
-      "ldr q6, [x28, #0x80]\n"
-      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       "ldr q7, [x28, #0x90]\n"
+      "sub x25, x25, #0x10\n"
+      ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45d  // ummla v29.4s, v2.16b, v8.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x6e89a45a  // ummla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45e  // ummla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
       ".inst 0x6e84a45b  // ummla v27.4s, v2.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
@@ -1035,43 +1033,42 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f946d  // udot v13.4s, v3.16b, v15.16b\n"
       "74:"  // Height 3: Multiply loop: unique 10: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
       "75:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x24, 84f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 84f\n"
+      "cmp x25, #0x8\n"
       "blt 78f\n"
       "76:"  // Height 3: Multiply loop: Odd block loop
-      "movi v7.16b, #0x0\n"
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x21], #0x8\n"
+      "ldr d3, [x22], #0x8\n"
       "trn1 v2.2d, v3.2d, v7.2d\n"
       "tbnz %x[flags], #31, 77f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "77:"  // Height 3: Multiply loop: unique 11: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
+      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a458  // ummla v24.4s, v2.16b, v8.16b\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
       "ldr q4, [x28, #0x30]\n"
-      ".inst 0x6e89a45c  // ummla v28.4s, v2.16b, v9.16b\n"
+      "sub x25, x25, #0x8\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x6e8aa459  // ummla v25.4s, v2.16b, v10.16b\n"
+      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x6e89a45c  // ummla v28.4s, v2.16b, v9.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x6e8aa459  // ummla v25.4s, v2.16b, v10.16b\n"
       ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e84a45d  // ummla v29.4s, v2.16b, v4.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x6e85a45a  // ummla v26.4s, v2.16b, v5.16b\n"
       ".inst 0x6e86a416  // ummla v22.4s, v0.16b, v6.16b\n"
@@ -1081,43 +1078,42 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e88a417  // ummla v23.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45f  // ummla v31.4s, v2.16b, v8.16b\n"
       "bge 76b\n"
-      "cbz x24, 84f\n"
       "78:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 80f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "ldr s3, [x21], #0x4\n"
-      "tbz x24, #1, 79f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "ld1 { v3.h }[2], [x21], #0x2\n"
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
-      "ld1 { v3.b }[6], [x21]\n"
+      "cbz x25, 84f\n"
+      "tbz x25, #2, 80f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "ldr s3, [x22], #0x4\n"
+      "tbz x25, #1, 79f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "ld1 { v3.h }[2], [x22], #0x2\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
+      "ld1 { v3.b }[6], [x22]\n"
       "b 82f\n"
       "79:"  // Height 3: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
-      "ld1 { v3.b }[4], [x21]\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
+      "ld1 { v3.b }[4], [x22]\n"
       "b 82f\n"
       "80:"  // Height 3: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 81f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "ldr h3, [x21], #0x2\n"
-      "tbz x24, #0, 82f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
-      "ld1 { v3.b }[2], [x21]\n"
+      "tbz x25, #1, 81f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "ldr h3, [x22], #0x2\n"
+      "tbz x25, #0, 82f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
+      "ld1 { v3.b }[2], [x22]\n"
       "b 82f\n"
       "81:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
-      "ldr b3, [x21, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
+      "ldr b3, [x22, #0x0]\n"
       "82:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v9.16b, #0x0\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v9.2d\n"
       "tbnz %x[flags], #31, 83f\n"
@@ -1125,24 +1121,24 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "83:"  // Height 3: Multiply loop: unique 12: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa458  // ummla v24.4s, v2.16b, v10.16b\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x6e84a414  // ummla v20.4s, v0.16b, v4.16b\n"
-      "ldr q7, [x28, #0x40]\n"
       ".inst 0x6e84a45c  // ummla v28.4s, v2.16b, v4.16b\n"
+      "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
       ".inst 0x6e85a411  // ummla v17.4s, v0.16b, v5.16b\n"
-      "ldr q9, [x28, #0x60]\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
+      "ldr q9, [x28, #0x60]\n"
       "ldr q10, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a45d  // ummla v29.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a412  // ummla v18.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a45a  // ummla v26.4s, v2.16b, v7.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x6e88a416  // ummla v22.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45e  // ummla v30.4s, v2.16b, v8.16b\n"
       ".inst 0x6e89a413  // ummla v19.4s, v0.16b, v9.16b\n"
@@ -1150,21 +1146,21 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8aa417  // ummla v23.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45f  // ummla v31.4s, v2.16b, v10.16b\n"
       "84:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 68b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
-      "add x20, x21, x19\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
@@ -1174,37 +1170,37 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "mov v31.16b, v4.16b\n"
       "tbnz %x[flags], #31, 85f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v3.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v3.4s }, [x22]\n"
       "addp v13.4s, v13.4s, v13.4s\n"
+      "neg v3.4s, v3.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v3.4s, v3.4s\n"
       "dup v13.4s, v13.s[0]\n"
       "mul v11.4s, v11.4s, v3.4s\n"
       "mul v12.4s, v12.4s, v3.4s\n"
       "mul v13.4s, v13.4s, v3.4s\n"
       "85:"  // Height 3: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v31.4s, v31.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v24.4s, v24.4s, v13.4s\n"
       "add v25.4s, v25.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v26.4s, v26.4s, v13.4s\n"
       "add v27.4s, v27.4s, v13.4s\n"
       "add v31.4s, v31.4s, v0.4s\n"
@@ -1236,98 +1232,98 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v20.16b, v0.16b\n"
       "and v6.16b, v21.16b, v0.16b\n"
+      "and v7.16b, v22.16b, v0.16b\n"
+      "and v8.16b, v16.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
       "sqadd v21.4s, v21.4s, v6.4s\n"
-      "and v7.16b, v22.16b, v0.16b\n"
-      "and v8.16b, v16.16b, v0.16b\n"
-      "and v9.16b, v17.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
       "sqadd v22.4s, v22.4s, v7.4s\n"
       "sqadd v16.4s, v16.4s, v8.4s\n"
-      "sqadd v17.4s, v17.4s, v9.4s\n"
+      "and v9.16b, v17.16b, v0.16b\n"
       "and v10.16b, v18.16b, v0.16b\n"
       "and v4.16b, v19.16b, v0.16b\n"
       "and v5.16b, v24.16b, v0.16b\n"
-      "sshr v10.4s, v10.4s, #0x1f\n"
-      "sshr v4.4s, v4.4s, #0x1f\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
-      "sqadd v18.4s, v18.4s, v10.4s\n"
-      "sqadd v19.4s, v19.4s, v4.4s\n"
-      "sqadd v24.4s, v24.4s, v5.4s\n"
       "and v6.16b, v25.16b, v0.16b\n"
       "and v7.16b, v26.16b, v0.16b\n"
       "and v8.16b, v27.16b, v0.16b\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
+      "sshr v10.4s, v10.4s, #0x1f\n"
+      "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
       "sshr v6.4s, v6.4s, #0x1f\n"
       "sshr v7.4s, v7.4s, #0x1f\n"
       "sshr v8.4s, v8.4s, #0x1f\n"
+      "sqadd v17.4s, v17.4s, v9.4s\n"
+      "sqadd v18.4s, v18.4s, v10.4s\n"
+      "sqadd v19.4s, v19.4s, v4.4s\n"
+      "sqadd v24.4s, v24.4s, v5.4s\n"
       "sqadd v25.4s, v25.4s, v6.4s\n"
       "sqadd v26.4s, v26.4s, v7.4s\n"
       "sqadd v27.4s, v27.4s, v8.4s\n"
       "86:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "uzp1 v31.8h, v31.8h, v20.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
-      "uzp1 v20.8h, v21.8h, v22.8h\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v31.8h, v31.8h, v20.8h\n"
+      "uzp1 v20.8h, v21.8h, v22.8h\n"
+      "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v24.8h, v24.8h, v25.8h\n"
       "uzp1 v25.8h, v26.8h, v27.8h\n"
@@ -1336,103 +1332,103 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 95f\n"
       "tbz x9, #3, 90f\n"
-      "str d31, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
+      "str d31, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
       "tbz x9, #2, 88f\n"
-      "st1 { v31.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v24.s }[2], [x20], #0x4\n"
+      "st1 { v31.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v24.s }[2], [x21], #0x4\n"
       "tbz x9, #1, 87f\n"
-      "st1 { v31.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v24.h }[6], [x20], #0x2\n"
+      "st1 { v31.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v24.h }[6], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v24.b }[14], [x20]\n"
+      "st1 { v31.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v24.b }[14], [x21]\n"
       "b 94f\n"
       "87:"  // Height 3: Partial direct writeback: partial_1_12
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v24.b }[12], [x20]\n"
+      "st1 { v31.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v24.b }[12], [x21]\n"
       "b 94f\n"
       "88:"  // Height 3: Partial direct writeback: partial_2_8
       "tbz x9, #1, 89f\n"
-      "st1 { v31.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v24.h }[4], [x20], #0x2\n"
+      "st1 { v31.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v24.h }[4], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v24.b }[10], [x20]\n"
+      "st1 { v31.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v24.b }[10], [x21]\n"
       "b 94f\n"
       "89:"  // Height 3: Partial direct writeback: partial_1_8
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v24.b }[8], [x20]\n"
+      "st1 { v31.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v24.b }[8], [x21]\n"
       "b 94f\n"
       "90:"  // Height 3: Partial direct writeback: partial_4_0
       "tbz x9, #2, 92f\n"
-      "str s31, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s24, [x20], #0x4\n"
+      "str s31, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s24, [x21], #0x4\n"
       "tbz x9, #1, 91f\n"
-      "st1 { v31.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v24.h }[2], [x20], #0x2\n"
+      "st1 { v31.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v24.h }[2], [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v24.b }[6], [x20]\n"
+      "st1 { v31.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v24.b }[6], [x21]\n"
       "b 94f\n"
       "91:"  // Height 3: Partial direct writeback: partial_1_4
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v24.b }[4], [x20]\n"
+      "st1 { v31.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v24.b }[4], [x21]\n"
       "b 94f\n"
       "92:"  // Height 3: Partial direct writeback: partial_2_0
       "tbz x9, #1, 93f\n"
-      "str h31, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h24, [x20], #0x2\n"
+      "str h31, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h24, [x21], #0x2\n"
       "tbz x9, #0, 94f\n"
-      "st1 { v31.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v24.b }[2], [x20]\n"
+      "st1 { v31.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v24.b }[2], [x21]\n"
       "b 94f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_0
-      "str b31, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b24, [x20, #0x0]\n"
+      "str b31, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b24, [x21, #0x0]\n"
       "94:"  // Height 3: Partial direct writeback: Done
       "b 96f\n"
       "95:"  // Height 3: Full writeback
-      "str q31, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
-      "str q24, [x20, #0x0]\n"
+      "str q31, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
+      "str q24, [x21, #0x0]\n"
       "96:"  // Height 3: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 66b\n"
       "b 130f\n"
       "97:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "movi v11.4s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "movi v12.4s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "movi v13.4s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "movi v14.4s, #0x0\n"
-      "mov x19, #0x4\n"
       "movi v15.16b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "98:"  // Height 4: Column loop
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
@@ -1451,70 +1447,70 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "99:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "100:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 101f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 102f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 102f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 102f\n"
       "101:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "102:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "blt 107f\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
-      "cmp x24, #0x20\n"
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
       "blt 105f\n"
       "103:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
+      "add x24, x24, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
-      "ldr q7, [x28, #0x20]\n"
+      "ldr q4, [x28, #0x60]\n"
       ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x40]\n"
       ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
-      "ldr q10, [x28, #0x50]\n"
-      "ldr q4, [x28, #0x60]\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
+      "ldr q7, [x28, #0x90]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
-      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e88a45d  // ummla v29.4s, v2.16b, v8.16b\n"
-      "ldr q7, [x28, #0x90]\n"
-      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x22, x22, #0x10\n"
+      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x6e89a45a  // ummla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45e  // ummla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
@@ -1548,52 +1544,52 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f946d  // udot v13.4s, v3.16b, v15.16b\n"
       "104:"  // Height 4: Multiply loop: unique 13: skip row sum
+      "ldr q1, [x24, #0x0]\n"
+      "ldr q2, [x23, #0x0]\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x20\n"
+      "ldr q3, [x22, #0x0]\n"
+      "ldr q4, [x21, #0x0]\n"
+      "ldr q5, [x28, #0x0]\n"
+      "ldr q6, [x28, #0x10]\n"
+      "ldr q7, [x28, #0x20]\n"
+      "ldr q8, [x28, #0x30]\n"
+      "ldr q9, [x28, #0x40]\n"
+      "ldr q10, [x28, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x24, x24, #0x10\n"
       "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x24, #0x20\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "ldr q1, [x23, #0x0]\n"
-      "ldr q2, [x22, #0x0]\n"
       "bge 103b\n"
       "105:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x21, #0x0]\n"
-      "sub x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
+      "sub x25, x25, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x28, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x28, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x6e85a410  // ummla v16.4s, v0.16b, v5.16b\n"
-      "ldr q7, [x28, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q4, [x28, #0x60]\n"
       ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      "ldr q8, [x28, #0x30]\n"
+      "ldr q5, [x28, #0x70]\n"
       ".inst 0x6e86a414  // ummla v20.4s, v0.16b, v6.16b\n"
-      "ldr q9, [x28, #0x40]\n"
       ".inst 0x6e86a45c  // ummla v28.4s, v2.16b, v6.16b\n"
-      "ldr q10, [x28, #0x50]\n"
-      "ldr q4, [x28, #0x60]\n"
+      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e87a411  // ummla v17.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
-      "ldr q5, [x28, #0x70]\n"
+      "ldr q7, [x28, #0x90]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e88a415  // ummla v21.4s, v0.16b, v8.16b\n"
-      "ldr q6, [x28, #0x80]\n"
       ".inst 0x6e88a45d  // ummla v29.4s, v2.16b, v8.16b\n"
-      "ldr q7, [x28, #0x90]\n"
-      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       "ldr q8, [x28, #0xa0]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0x6e89a412  // ummla v18.4s, v0.16b, v9.16b\n"
       ".inst 0x6e89a45a  // ummla v26.4s, v2.16b, v9.16b\n"
       "ldr q9, [x28, #0xb0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e8aa416  // ummla v22.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45e  // ummla v30.4s, v2.16b, v10.16b\n"
       "ldr q10, [x28, #0xc0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6e84a413  // ummla v19.4s, v0.16b, v4.16b\n"
       ".inst 0x6e84a45b  // ummla v27.4s, v2.16b, v4.16b\n"
       "ldr q4, [x28, #0xd0]\n"
@@ -1624,44 +1620,44 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f942b  // udot v11.4s, v1.16b, v15.16b\n"
       ".inst 0x6e8f946d  // udot v13.4s, v3.16b, v15.16b\n"
       "106:"  // Height 4: Multiply loop: unique 14: skip row sum
+      "prfm pldl1keep, [x24, #0x80]\n"
       "prfm pldl1keep, [x23, #0x80]\n"
       "prfm pldl1keep, [x22, #0x80]\n"
       "prfm pldl1keep, [x21, #0x80]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       "107:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x24, 116f\n"
-      "cmp x24, #0x8\n"
+      "cbz x25, 116f\n"
+      "cmp x25, #0x8\n"
       "blt 110f\n"
       "108:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x23], #0x8\n"
-      "ldr d2, [x22], #0x8\n"
+      "ldr d1, [x24], #0x8\n"
+      "ldr d2, [x23], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "ldr d3, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v2.2d, v3.2d, v7.2d\n"
       "tbnz %x[flags], #31, 109f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "109:"  // Height 4: Multiply loop: unique 15: skip row sum
       "ldr q8, [x28, #0x0]\n"
-      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       "ldr q9, [x28, #0x10]\n"
-      "sub x24, x24, #0x8\n"
+      ".inst 0x6e88a410  // ummla v16.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a458  // ummla v24.4s, v2.16b, v8.16b\n"
       "ldr q10, [x28, #0x20]\n"
-      "cmp x24, #0x8\n"
-      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
       "ldr q4, [x28, #0x30]\n"
-      ".inst 0x6e89a45c  // ummla v28.4s, v2.16b, v9.16b\n"
+      "sub x25, x25, #0x8\n"
+      "cmp x25, #0x8\n"
       "ldr q5, [x28, #0x40]\n"
-      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
       "ldr q6, [x28, #0x50]\n"
-      ".inst 0x6e8aa459  // ummla v25.4s, v2.16b, v10.16b\n"
+      ".inst 0x6e89a414  // ummla v20.4s, v0.16b, v9.16b\n"
+      ".inst 0x6e89a45c  // ummla v28.4s, v2.16b, v9.16b\n"
       "ldr q7, [x28, #0x60]\n"
       "ldr q8, [x28, #0x70]\n"
+      ".inst 0x6e8aa411  // ummla v17.4s, v0.16b, v10.16b\n"
+      ".inst 0x6e8aa459  // ummla v25.4s, v2.16b, v10.16b\n"
       ".inst 0x6e84a415  // ummla v21.4s, v0.16b, v4.16b\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e84a45d  // ummla v29.4s, v2.16b, v4.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x6e85a412  // ummla v18.4s, v0.16b, v5.16b\n"
       ".inst 0x6e85a45a  // ummla v26.4s, v2.16b, v5.16b\n"
       ".inst 0x6e86a416  // ummla v22.4s, v0.16b, v6.16b\n"
@@ -1671,48 +1667,48 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e88a417  // ummla v23.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45f  // ummla v31.4s, v2.16b, v8.16b\n"
       "bge 108b\n"
-      "cbz x24, 116f\n"
       "110:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x24, #2, 112f\n"
-      "ldr s1, [x23], #0x4\n"
-      "ldr s2, [x22], #0x4\n"
-      "ldr s3, [x21], #0x4\n"
-      "ldr s9, [x20], #0x4\n"
-      "tbz x24, #1, 111f\n"
-      "ld1 { v1.h }[2], [x23], #0x2\n"
-      "ld1 { v2.h }[2], [x22], #0x2\n"
-      "ld1 { v3.h }[2], [x21], #0x2\n"
-      "ld1 { v9.h }[2], [x20], #0x2\n"
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[6], [x23]\n"
-      "ld1 { v2.b }[6], [x22]\n"
-      "ld1 { v3.b }[6], [x21]\n"
-      "ld1 { v9.b }[6], [x20]\n"
+      "cbz x25, 116f\n"
+      "tbz x25, #2, 112f\n"
+      "ldr s1, [x24], #0x4\n"
+      "ldr s2, [x23], #0x4\n"
+      "ldr s3, [x22], #0x4\n"
+      "ldr s9, [x21], #0x4\n"
+      "tbz x25, #1, 111f\n"
+      "ld1 { v1.h }[2], [x24], #0x2\n"
+      "ld1 { v2.h }[2], [x23], #0x2\n"
+      "ld1 { v3.h }[2], [x22], #0x2\n"
+      "ld1 { v9.h }[2], [x21], #0x2\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[6], [x24]\n"
+      "ld1 { v2.b }[6], [x23]\n"
+      "ld1 { v3.b }[6], [x22]\n"
+      "ld1 { v9.b }[6], [x21]\n"
       "b 114f\n"
       "111:"  // Height 4: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[4], [x23]\n"
-      "ld1 { v2.b }[4], [x22]\n"
-      "ld1 { v3.b }[4], [x21]\n"
-      "ld1 { v9.b }[4], [x20]\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[4], [x24]\n"
+      "ld1 { v2.b }[4], [x23]\n"
+      "ld1 { v3.b }[4], [x22]\n"
+      "ld1 { v9.b }[4], [x21]\n"
       "b 114f\n"
       "112:"  // Height 4: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x24, #1, 113f\n"
-      "ldr h1, [x23], #0x2\n"
-      "ldr h2, [x22], #0x2\n"
-      "ldr h3, [x21], #0x2\n"
-      "ldr h9, [x20], #0x2\n"
-      "tbz x24, #0, 114f\n"
-      "ld1 { v1.b }[2], [x23]\n"
-      "ld1 { v2.b }[2], [x22]\n"
-      "ld1 { v3.b }[2], [x21]\n"
-      "ld1 { v9.b }[2], [x20]\n"
+      "tbz x25, #1, 113f\n"
+      "ldr h1, [x24], #0x2\n"
+      "ldr h2, [x23], #0x2\n"
+      "ldr h3, [x22], #0x2\n"
+      "ldr h9, [x21], #0x2\n"
+      "tbz x25, #0, 114f\n"
+      "ld1 { v1.b }[2], [x24]\n"
+      "ld1 { v2.b }[2], [x23]\n"
+      "ld1 { v3.b }[2], [x22]\n"
+      "ld1 { v9.b }[2], [x21]\n"
       "b 114f\n"
       "113:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x23, #0x0]\n"
-      "ldr b2, [x22, #0x0]\n"
-      "ldr b3, [x21, #0x0]\n"
-      "ldr b9, [x20, #0x0]\n"
+      "ldr b1, [x24, #0x0]\n"
+      "ldr b2, [x23, #0x0]\n"
+      "ldr b3, [x22, #0x0]\n"
+      "ldr b9, [x21, #0x0]\n"
       "114:"  // Height 4: Multiply loop: Ragged operand read: Done
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v9.2d\n"
@@ -1721,24 +1717,24 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8f944d  // udot v13.4s, v2.16b, v15.16b\n"
       "115:"  // Height 4: Multiply loop: unique 16: skip row sum
       "ldr q10, [x28, #0x0]\n"
-      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       "ldr q4, [x28, #0x10]\n"
+      ".inst 0x6e8aa410  // ummla v16.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa458  // ummla v24.4s, v2.16b, v10.16b\n"
       "ldr q5, [x28, #0x20]\n"
       "ldr q6, [x28, #0x30]\n"
       ".inst 0x6e84a414  // ummla v20.4s, v0.16b, v4.16b\n"
-      "ldr q7, [x28, #0x40]\n"
       ".inst 0x6e84a45c  // ummla v28.4s, v2.16b, v4.16b\n"
+      "ldr q7, [x28, #0x40]\n"
       "ldr q8, [x28, #0x50]\n"
       ".inst 0x6e85a411  // ummla v17.4s, v0.16b, v5.16b\n"
-      "ldr q9, [x28, #0x60]\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
+      "ldr q9, [x28, #0x60]\n"
       "ldr q10, [x28, #0x70]\n"
-      "add x28, x28, #0x80\n"
       ".inst 0x6e86a415  // ummla v21.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a45d  // ummla v29.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a412  // ummla v18.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a45a  // ummla v26.4s, v2.16b, v7.16b\n"
+      "add x28, x28, #0x80\n"
       ".inst 0x6e88a416  // ummla v22.4s, v0.16b, v8.16b\n"
       ".inst 0x6e88a45e  // ummla v30.4s, v2.16b, v8.16b\n"
       ".inst 0x6e89a413  // ummla v19.4s, v0.16b, v9.16b\n"
@@ -1746,25 +1742,25 @@ void a64_hybrid_u8qa_mmla_4x16 (
       ".inst 0x6e8aa417  // ummla v23.4s, v0.16b, v10.16b\n"
       ".inst 0x6e8aa45f  // ummla v31.4s, v2.16b, v10.16b\n"
       "116:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 100b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 v4.2d, v16.2d, v20.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x26, #0x0]\n"
-      "add x21, x26, x19\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x27, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
-      "add x20, x21, x19\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "uzp1 v22.2d, v19.2d, v23.2d\n"
+      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "uzp1 v23.2d, v24.2d, v28.2d\n"
       "uzp2 v24.2d, v24.2d, v28.2d\n"
@@ -1776,13 +1772,13 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "mov v31.16b, v4.16b\n"
       "tbnz %x[flags], #31, 117f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "addp v11.4s, v11.4s, v11.4s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "addp v13.4s, v13.4s, v13.4s\n"
+      "neg v4.4s, v4.4s\n"
       "dup v12.4s, v11.s[3]\n"
       "dup v11.4s, v11.s[0]\n"
-      "neg v4.4s, v4.4s\n"
       "dup v14.4s, v13.s[3]\n"
       "dup v13.4s, v13.s[0]\n"
       "mul v11.4s, v11.4s, v4.4s\n"
@@ -1790,25 +1786,25 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "mul v13.4s, v13.4s, v4.4s\n"
       "mul v14.4s, v14.4s, v4.4s\n"
       "117:"  // Height 4: skip row sum fixup
+      "ldr q0, [x10, #0x0]\n"
+      "ldr q1, [x10, #0x10]\n"
       "add v31.4s, v31.4s, v11.4s\n"
-      "ldr q0, [x27, #0x0]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add v20.4s, v20.4s, v11.4s\n"
-      "ldr q1, [x27, #0x10]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ldr q2, [x10, #0x20]\n"
+      "ldr q3, [x10, #0x30]\n"
       "add v21.4s, v21.4s, v11.4s\n"
-      "ldr q2, [x27, #0x20]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add v22.4s, v22.4s, v11.4s\n"
-      "ldr q3, [x27, #0x30]\n"
-      "add x27, x27, #0x40\n"
       "add v16.4s, v16.4s, v12.4s\n"
-      "ld1r { v4.4s }, [x22]\n"
       "add v17.4s, v17.4s, v12.4s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "add v18.4s, v18.4s, v12.4s\n"
       "add v19.4s, v19.4s, v12.4s\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add v23.4s, v23.4s, v13.4s\n"
       "add v28.4s, v28.4s, v13.4s\n"
+      "add x10, x10, #0x40\n"
       "add v29.4s, v29.4s, v13.4s\n"
       "add v30.4s, v30.4s, v13.4s\n"
       "add v24.4s, v24.4s, v14.4s\n"
@@ -1851,126 +1847,126 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "tbz %x[flags], #5, 118f\n"
       "and v4.16b, v31.16b, v0.16b\n"
       "and v5.16b, v20.16b, v0.16b\n"
-      "and v6.16b, v21.16b, v0.16b\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
       "sqadd v31.4s, v31.4s, v4.4s\n"
       "sqadd v20.4s, v20.4s, v5.4s\n"
-      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "and v6.16b, v21.16b, v0.16b\n"
       "and v7.16b, v22.16b, v0.16b\n"
       "and v8.16b, v16.16b, v0.16b\n"
       "and v9.16b, v17.16b, v0.16b\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sshr v9.4s, v9.4s, #0x1f\n"
-      "sqadd v22.4s, v22.4s, v7.4s\n"
-      "sqadd v16.4s, v16.4s, v8.4s\n"
-      "sqadd v17.4s, v17.4s, v9.4s\n"
       "and v10.16b, v18.16b, v0.16b\n"
       "and v4.16b, v19.16b, v0.16b\n"
       "and v5.16b, v23.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
+      "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
       "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v21.4s, v21.4s, v6.4s\n"
+      "sqadd v22.4s, v22.4s, v7.4s\n"
+      "sqadd v16.4s, v16.4s, v8.4s\n"
+      "sqadd v17.4s, v17.4s, v9.4s\n"
       "sqadd v18.4s, v18.4s, v10.4s\n"
       "sqadd v19.4s, v19.4s, v4.4s\n"
       "sqadd v23.4s, v23.4s, v5.4s\n"
       "and v6.16b, v28.16b, v0.16b\n"
       "and v7.16b, v29.16b, v0.16b\n"
       "and v8.16b, v30.16b, v0.16b\n"
-      "sshr v6.4s, v6.4s, #0x1f\n"
-      "sshr v7.4s, v7.4s, #0x1f\n"
-      "sshr v8.4s, v8.4s, #0x1f\n"
-      "sqadd v28.4s, v28.4s, v6.4s\n"
-      "sqadd v29.4s, v29.4s, v7.4s\n"
-      "sqadd v30.4s, v30.4s, v8.4s\n"
       "and v9.16b, v24.16b, v0.16b\n"
       "and v10.16b, v25.16b, v0.16b\n"
       "and v4.16b, v26.16b, v0.16b\n"
+      "and v5.16b, v27.16b, v0.16b\n"
+      "sshr v6.4s, v6.4s, #0x1f\n"
+      "sshr v7.4s, v7.4s, #0x1f\n"
+      "sshr v8.4s, v8.4s, #0x1f\n"
       "sshr v9.4s, v9.4s, #0x1f\n"
       "sshr v10.4s, v10.4s, #0x1f\n"
       "sshr v4.4s, v4.4s, #0x1f\n"
+      "sshr v5.4s, v5.4s, #0x1f\n"
+      "sqadd v28.4s, v28.4s, v6.4s\n"
+      "sqadd v29.4s, v29.4s, v7.4s\n"
+      "sqadd v30.4s, v30.4s, v8.4s\n"
       "sqadd v24.4s, v24.4s, v9.4s\n"
       "sqadd v25.4s, v25.4s, v10.4s\n"
       "sqadd v26.4s, v26.4s, v4.4s\n"
-      "and v5.16b, v27.16b, v0.16b\n"
-      "sshr v5.4s, v5.4s, #0x1f\n"
       "sqadd v27.4s, v27.4s, v5.4s\n"
       "118:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1r { v4.4s }, [x23]\n"
       "srshl v31.4s, v31.4s, v0.4s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1r { v4.4s }, [x22]\n"
       "srshl v20.4s, v20.4s, v0.4s\n"
-      "add x22, %x[qp], %[minval]\n"
       "srshl v21.4s, v21.4s, v0.4s\n"
-      "ld1r { v5.4s }, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
       "srshl v22.4s, v22.4s, v0.4s\n"
-      "ld1r { v6.4s }, [x22]\n"
-      "cmp x9, #0x10\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1r { v6.4s }, [x23]\n"
       "srshl v16.4s, v16.4s, v0.4s\n"
       "srshl v17.4s, v17.4s, v0.4s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1r { v5.4s }, [x23]\n"
+      "srshl v18.4s, v18.4s, v0.4s\n"
+      "srshl v19.4s, v19.4s, v0.4s\n"
+      "cmp x9, #0x10\n"
+      "srshl v23.4s, v23.4s, v0.4s\n"
+      "srshl v28.4s, v28.4s, v0.4s\n"
+      "srshl v29.4s, v29.4s, v0.4s\n"
+      "srshl v30.4s, v30.4s, v0.4s\n"
+      "srshl v24.4s, v24.4s, v0.4s\n"
+      "srshl v25.4s, v25.4s, v0.4s\n"
+      "srshl v26.4s, v26.4s, v0.4s\n"
+      "srshl v27.4s, v27.4s, v0.4s\n"
       "add v31.4s, v31.4s, v4.4s\n"
       "add v20.4s, v20.4s, v4.4s\n"
       "add v21.4s, v21.4s, v4.4s\n"
-      "smin v31.4s, v31.4s, v6.4s\n"
-      "smin v20.4s, v20.4s, v6.4s\n"
-      "smin v21.4s, v21.4s, v6.4s\n"
-      "smax v31.4s, v31.4s, v5.4s\n"
-      "smax v20.4s, v20.4s, v5.4s\n"
-      "smax v21.4s, v21.4s, v5.4s\n"
       "add v22.4s, v22.4s, v4.4s\n"
       "add v16.4s, v16.4s, v4.4s\n"
       "add v17.4s, v17.4s, v4.4s\n"
-      "smin v22.4s, v22.4s, v6.4s\n"
-      "smin v16.4s, v16.4s, v6.4s\n"
-      "smin v17.4s, v17.4s, v6.4s\n"
-      "smax v22.4s, v22.4s, v5.4s\n"
-      "smax v16.4s, v16.4s, v5.4s\n"
-      "smax v17.4s, v17.4s, v5.4s\n"
-      "srshl v18.4s, v18.4s, v0.4s\n"
-      "srshl v19.4s, v19.4s, v0.4s\n"
-      "srshl v23.4s, v23.4s, v0.4s\n"
-      "srshl v28.4s, v28.4s, v0.4s\n"
       "add v18.4s, v18.4s, v4.4s\n"
       "add v19.4s, v19.4s, v4.4s\n"
       "add v23.4s, v23.4s, v4.4s\n"
+      "add v28.4s, v28.4s, v4.4s\n"
+      "add v29.4s, v29.4s, v4.4s\n"
+      "add v30.4s, v30.4s, v4.4s\n"
+      "add v24.4s, v24.4s, v4.4s\n"
+      "add v25.4s, v25.4s, v4.4s\n"
+      "add v26.4s, v26.4s, v4.4s\n"
+      "add v27.4s, v27.4s, v4.4s\n"
+      "smin v31.4s, v31.4s, v6.4s\n"
+      "smin v20.4s, v20.4s, v6.4s\n"
+      "smin v21.4s, v21.4s, v6.4s\n"
+      "smin v22.4s, v22.4s, v6.4s\n"
+      "smin v16.4s, v16.4s, v6.4s\n"
+      "smin v17.4s, v17.4s, v6.4s\n"
       "smin v18.4s, v18.4s, v6.4s\n"
       "smin v19.4s, v19.4s, v6.4s\n"
       "smin v23.4s, v23.4s, v6.4s\n"
+      "smin v28.4s, v28.4s, v6.4s\n"
+      "smin v29.4s, v29.4s, v6.4s\n"
+      "smin v30.4s, v30.4s, v6.4s\n"
+      "smin v24.4s, v24.4s, v6.4s\n"
+      "smin v25.4s, v25.4s, v6.4s\n"
+      "smin v26.4s, v26.4s, v6.4s\n"
+      "smin v27.4s, v27.4s, v6.4s\n"
+      "smax v31.4s, v31.4s, v5.4s\n"
+      "smax v20.4s, v20.4s, v5.4s\n"
+      "smax v21.4s, v21.4s, v5.4s\n"
+      "smax v22.4s, v22.4s, v5.4s\n"
+      "smax v16.4s, v16.4s, v5.4s\n"
+      "smax v17.4s, v17.4s, v5.4s\n"
       "smax v18.4s, v18.4s, v5.4s\n"
       "smax v19.4s, v19.4s, v5.4s\n"
       "smax v23.4s, v23.4s, v5.4s\n"
-      "add v28.4s, v28.4s, v4.4s\n"
-      "srshl v29.4s, v29.4s, v0.4s\n"
-      "srshl v30.4s, v30.4s, v0.4s\n"
-      "smin v28.4s, v28.4s, v6.4s\n"
-      "srshl v24.4s, v24.4s, v0.4s\n"
-      "add v29.4s, v29.4s, v4.4s\n"
       "smax v28.4s, v28.4s, v5.4s\n"
-      "add v30.4s, v30.4s, v4.4s\n"
-      "smin v29.4s, v29.4s, v6.4s\n"
-      "add v24.4s, v24.4s, v4.4s\n"
-      "smin v30.4s, v30.4s, v6.4s\n"
       "smax v29.4s, v29.4s, v5.4s\n"
-      "smin v24.4s, v24.4s, v6.4s\n"
       "smax v30.4s, v30.4s, v5.4s\n"
-      "srshl v25.4s, v25.4s, v0.4s\n"
       "smax v24.4s, v24.4s, v5.4s\n"
-      "srshl v26.4s, v26.4s, v0.4s\n"
-      "srshl v27.4s, v27.4s, v0.4s\n"
-      "add v25.4s, v25.4s, v4.4s\n"
-      "uzp1 v31.8h, v31.8h, v20.8h\n"
-      "add v26.4s, v26.4s, v4.4s\n"
-      "smin v25.4s, v25.4s, v6.4s\n"
-      "add v27.4s, v27.4s, v4.4s\n"
-      "smin v26.4s, v26.4s, v6.4s\n"
       "smax v25.4s, v25.4s, v5.4s\n"
-      "smin v27.4s, v27.4s, v6.4s\n"
       "smax v26.4s, v26.4s, v5.4s\n"
-      "uzp1 v20.8h, v21.8h, v22.8h\n"
       "smax v27.4s, v27.4s, v5.4s\n"
+      "uzp1 v31.8h, v31.8h, v20.8h\n"
+      "uzp1 v20.8h, v21.8h, v22.8h\n"
       "uzp1 v16.8h, v16.8h, v17.8h\n"
       "uzp1 v17.8h, v18.8h, v19.8h\n"
       "uzp1 v23.8h, v23.8h, v28.8h\n"
@@ -1983,120 +1979,120 @@ void a64_hybrid_u8qa_mmla_4x16 (
       "uzp1 v24.16b, v24.16b, v25.16b\n"
       "bge 127f\n"
       "tbz x9, #3, 122f\n"
-      "str d31, [x26], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "str d24, [x19], #0x8\n"
+      "str d31, [x27], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "str d24, [x20], #0x8\n"
       "tbz x9, #2, 120f\n"
-      "st1 { v31.s }[2], [x26], #0x4\n"
-      "st1 { v16.s }[2], [x21], #0x4\n"
-      "st1 { v23.s }[2], [x20], #0x4\n"
-      "st1 { v24.s }[2], [x19], #0x4\n"
+      "st1 { v31.s }[2], [x27], #0x4\n"
+      "st1 { v16.s }[2], [x22], #0x4\n"
+      "st1 { v23.s }[2], [x21], #0x4\n"
+      "st1 { v24.s }[2], [x20], #0x4\n"
       "tbz x9, #1, 119f\n"
-      "st1 { v31.h }[6], [x26], #0x2\n"
-      "st1 { v16.h }[6], [x21], #0x2\n"
-      "st1 { v23.h }[6], [x20], #0x2\n"
-      "st1 { v24.h }[6], [x19], #0x2\n"
+      "st1 { v31.h }[6], [x27], #0x2\n"
+      "st1 { v16.h }[6], [x22], #0x2\n"
+      "st1 { v23.h }[6], [x21], #0x2\n"
+      "st1 { v24.h }[6], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[14], [x26]\n"
-      "st1 { v16.b }[14], [x21]\n"
-      "st1 { v23.b }[14], [x20]\n"
-      "st1 { v24.b }[14], [x19]\n"
+      "st1 { v31.b }[14], [x27]\n"
+      "st1 { v16.b }[14], [x22]\n"
+      "st1 { v23.b }[14], [x21]\n"
+      "st1 { v24.b }[14], [x20]\n"
       "b 126f\n"
       "119:"  // Height 4: Partial direct writeback: partial_1_12
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[12], [x26]\n"
-      "st1 { v16.b }[12], [x21]\n"
-      "st1 { v23.b }[12], [x20]\n"
-      "st1 { v24.b }[12], [x19]\n"
+      "st1 { v31.b }[12], [x27]\n"
+      "st1 { v16.b }[12], [x22]\n"
+      "st1 { v23.b }[12], [x21]\n"
+      "st1 { v24.b }[12], [x20]\n"
       "b 126f\n"
       "120:"  // Height 4: Partial direct writeback: partial_2_8
       "tbz x9, #1, 121f\n"
-      "st1 { v31.h }[4], [x26], #0x2\n"
-      "st1 { v16.h }[4], [x21], #0x2\n"
-      "st1 { v23.h }[4], [x20], #0x2\n"
-      "st1 { v24.h }[4], [x19], #0x2\n"
+      "st1 { v31.h }[4], [x27], #0x2\n"
+      "st1 { v16.h }[4], [x22], #0x2\n"
+      "st1 { v23.h }[4], [x21], #0x2\n"
+      "st1 { v24.h }[4], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[10], [x26]\n"
-      "st1 { v16.b }[10], [x21]\n"
-      "st1 { v23.b }[10], [x20]\n"
-      "st1 { v24.b }[10], [x19]\n"
+      "st1 { v31.b }[10], [x27]\n"
+      "st1 { v16.b }[10], [x22]\n"
+      "st1 { v23.b }[10], [x21]\n"
+      "st1 { v24.b }[10], [x20]\n"
       "b 126f\n"
       "121:"  // Height 4: Partial direct writeback: partial_1_8
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[8], [x26]\n"
-      "st1 { v16.b }[8], [x21]\n"
-      "st1 { v23.b }[8], [x20]\n"
-      "st1 { v24.b }[8], [x19]\n"
+      "st1 { v31.b }[8], [x27]\n"
+      "st1 { v16.b }[8], [x22]\n"
+      "st1 { v23.b }[8], [x21]\n"
+      "st1 { v24.b }[8], [x20]\n"
       "b 126f\n"
       "122:"  // Height 4: Partial direct writeback: partial_4_0
       "tbz x9, #2, 124f\n"
-      "str s31, [x26], #0x4\n"
-      "str s16, [x21], #0x4\n"
-      "str s23, [x20], #0x4\n"
-      "str s24, [x19], #0x4\n"
+      "str s31, [x27], #0x4\n"
+      "str s16, [x22], #0x4\n"
+      "str s23, [x21], #0x4\n"
+      "str s24, [x20], #0x4\n"
       "tbz x9, #1, 123f\n"
-      "st1 { v31.h }[2], [x26], #0x2\n"
-      "st1 { v16.h }[2], [x21], #0x2\n"
-      "st1 { v23.h }[2], [x20], #0x2\n"
-      "st1 { v24.h }[2], [x19], #0x2\n"
+      "st1 { v31.h }[2], [x27], #0x2\n"
+      "st1 { v16.h }[2], [x22], #0x2\n"
+      "st1 { v23.h }[2], [x21], #0x2\n"
+      "st1 { v24.h }[2], [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[6], [x26]\n"
-      "st1 { v16.b }[6], [x21]\n"
-      "st1 { v23.b }[6], [x20]\n"
-      "st1 { v24.b }[6], [x19]\n"
+      "st1 { v31.b }[6], [x27]\n"
+      "st1 { v16.b }[6], [x22]\n"
+      "st1 { v23.b }[6], [x21]\n"
+      "st1 { v24.b }[6], [x20]\n"
       "b 126f\n"
       "123:"  // Height 4: Partial direct writeback: partial_1_4
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[4], [x26]\n"
-      "st1 { v16.b }[4], [x21]\n"
-      "st1 { v23.b }[4], [x20]\n"
-      "st1 { v24.b }[4], [x19]\n"
+      "st1 { v31.b }[4], [x27]\n"
+      "st1 { v16.b }[4], [x22]\n"
+      "st1 { v23.b }[4], [x21]\n"
+      "st1 { v24.b }[4], [x20]\n"
       "b 126f\n"
       "124:"  // Height 4: Partial direct writeback: partial_2_0
       "tbz x9, #1, 125f\n"
-      "str h31, [x26], #0x2\n"
-      "str h16, [x21], #0x2\n"
-      "str h23, [x20], #0x2\n"
-      "str h24, [x19], #0x2\n"
+      "str h31, [x27], #0x2\n"
+      "str h16, [x22], #0x2\n"
+      "str h23, [x21], #0x2\n"
+      "str h24, [x20], #0x2\n"
       "tbz x9, #0, 126f\n"
-      "st1 { v31.b }[2], [x26]\n"
-      "st1 { v16.b }[2], [x21]\n"
-      "st1 { v23.b }[2], [x20]\n"
-      "st1 { v24.b }[2], [x19]\n"
+      "st1 { v31.b }[2], [x27]\n"
+      "st1 { v16.b }[2], [x22]\n"
+      "st1 { v23.b }[2], [x21]\n"
+      "st1 { v24.b }[2], [x20]\n"
       "b 126f\n"
       "125:"  // Height 4: Partial direct writeback: partial_1_0
-      "str b31, [x26, #0x0]\n"
-      "str b16, [x21, #0x0]\n"
-      "str b23, [x20, #0x0]\n"
-      "str b24, [x19, #0x0]\n"
+      "str b31, [x27, #0x0]\n"
+      "str b16, [x22, #0x0]\n"
+      "str b23, [x21, #0x0]\n"
+      "str b24, [x20, #0x0]\n"
       "126:"  // Height 4: Partial direct writeback: Done
       "b 128f\n"
       "127:"  // Height 4: Full writeback
-      "str q31, [x26, #0x0]\n"
-      "add x26, x26, #0x10\n"
-      "str q16, [x21, #0x0]\n"
-      "str q23, [x20, #0x0]\n"
-      "str q24, [x19, #0x0]\n"
+      "str q31, [x27, #0x0]\n"
+      "add x27, x27, #0x10\n"
+      "str q16, [x22, #0x0]\n"
+      "str q23, [x21, #0x0]\n"
+      "str q24, [x20, #0x0]\n"
       "128:"  // Height 4: Writeback done
       "subs x9, x9, #0x10\n"
       "bgt 98b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 130f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 129f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "129:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "130:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp
index 8833651768..705f6525b6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "cmp %x[M], #0x2\n"
       "bgt 69f\n"
       "beq 35f\n"
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x15, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 12f\n"
-      "cmp x17, #0x10\n"
+      "cmp x8, #0x10\n"
       "bge 11f\n"
-      "tbz x17, #3, 6f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "tbz x17, #2, 4f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v11.s }[2], [x15]\n"
+      "tbz x8, #3, 6f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "tbz x8, #2, 4f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 3f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v11.s }[2], [x17]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s11, [x15, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s11, [x17, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x17, #1, 5f\n"
-      "ldr d10, [x15], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v10.s }[2], [x15]\n"
+      "tbz x8, #1, 5f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v10.s }[2], [x17]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s10, [x15, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s10, [x17, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x17, #2, 8f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 7f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v9.s }[2], [x15]\n"
+      "tbz x8, #2, 8f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 7f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v9.s }[2], [x17]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 10f\n"
-      "ldr s9, [x15, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 10f\n"
+      "ldr s9, [x17, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x17, #1, 9f\n"
-      "ldr d8, [x15], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x17, #0, 10f\n"
-      "ld1 { v8.s }[2], [x15]\n"
+      "tbz x8, #1, 9f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x8, #0, 10f\n"
+      "ld1 { v8.s }[2], [x17]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 13f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
       "b 13f\n"
       "12:"  // Height 1: no accumulate
       "movi v8.4s, #0x0\n"
@@ -161,112 +161,109 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
       "13:"  // Height 1: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "14:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 15f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "cbnz x14, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "cbnz x15, 16f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
       "b 16f\n"
       "15:"  // Height 1: setup direct input
-      "mov x12, %x[input_ptr]\n"
+      "mov x13, %x[input_ptr]\n"
       "16:"  // Height 1: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 19f\n"
-      "ldr q0, [x12, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
       "ldr q6, [x16, #0x0]\n"
-      "cmp x13, #0x20\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 18f\n"
       "17:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
-      "ldr x11, [x16, #0x18]\n"
-      "add x12, x12, #0x10\n"
       "ldr d6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "ldr x10, [x16, #0x28]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
-      "ldr x10, [x16, #0x48]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
-      "ldr x10, [x16, #0x68]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x78]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
-      "ldr x10, [x16, #0x88]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x98]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
-      "ldr x10, [x16, #0xa8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xb8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
-      "ldr x10, [x16, #0xc8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
-      "ldr x10, [x16, #0xe8]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xf0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xf8]\n"
+      "mov v7.d[1], x11\n"
+      "add x13, x13, #0x10\n"
       "add x16, x16, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
-      "ldr x10, [x16, #0x8]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d0, [x12, #0x0]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d0, [x13, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       "bge 17b\n"
       "18:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
       "ldr q6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
+      ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
@@ -290,200 +287,203 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "ldr q6, [x16, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       "ldr q7, [x16, #0xf0]\n"
+      "add x13, x13, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
-      "add x16, x16, #0x100\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
+      "add x16, x16, #0x100\n"
       "19:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x13, 24f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 24f\n"
+      "cmp x14, #0x4\n"
       "blt 21f\n"
       "20:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "cmp x13, #0x4\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x16, #0x20]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
+      "ldr q6, [x16, #0x20]\n"
+      "cmp x14, #0x4\n"
       "ldr q7, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x16, x16, #0x40\n"
       "bge 20b\n"
-      "cbz x13, 24f\n"
       "21:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 22f\n"
-      "ldr h0, [x12], #0x2\n"
-      "tbz x13, #0, 23f\n"
-      "ld1 { v0.b }[2], [x12]\n"
+      "cbz x14, 24f\n"
+      "tbz x14, #1, 22f\n"
+      "ldr h0, [x13], #0x2\n"
+      "tbz x14, #0, 23f\n"
+      "ld1 { v0.b }[2], [x13]\n"
       "b 23f\n"
       "22:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q6, [x16, #0x20]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x30]\n"
-      "add x16, x16, #0x40\n"
+      "ldr q6, [x16, #0x20]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x16, x16, #0x40\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 14b\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
       "bge 33f\n"
-      "tbz x17, #3, 28f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "tbz x17, #2, 26f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 25f\n"
-      "str d11, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v11.s }[2], [x15]\n"
+      "tbz x8, #3, 28f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "tbz x8, #2, 26f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 25f\n"
+      "str d11, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v11.s }[2], [x17]\n"
       "b 32f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 32f\n"
-      "str s11, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s11, [x17, #0x0]\n"
       "b 32f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 27f\n"
-      "str d10, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v10.s }[2], [x15]\n"
+      "tbz x8, #1, 27f\n"
+      "str d10, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v10.s }[2], [x17]\n"
       "b 32f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 32f\n"
-      "str s10, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s10, [x17, #0x0]\n"
       "b 32f\n"
       "28:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 30f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "tbz x17, #1, 29f\n"
-      "str d9, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v9.s }[2], [x15]\n"
+      "tbz x8, #2, 30f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "tbz x8, #1, 29f\n"
+      "str d9, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v9.s }[2], [x17]\n"
       "b 32f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 32f\n"
-      "str s9, [x15, #0x0]\n"
+      "tbz x8, #0, 32f\n"
+      "str s9, [x17, #0x0]\n"
       "b 32f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 31f\n"
-      "str d8, [x15], #0x8\n"
-      "tbz x17, #0, 32f\n"
-      "st1 { v8.s }[2], [x15]\n"
+      "tbz x8, #1, 31f\n"
+      "str d8, [x17], #0x8\n"
+      "tbz x8, #0, 32f\n"
+      "st1 { v8.s }[2], [x17]\n"
       "b 32f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
       "32:"  // Height 1: Partial direct writeback: Done
       "b 34f\n"
       "33:"  // Height 1: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
       "34:"  // Height 1: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 2b\n"
       "b 206f\n"
       "35:"  // Height 2
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "36:"  // Height 2: Column loop
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x8, #0x10\n"
+      "add x24, x17, x20, LSL #2\n"
       "bge 45f\n"
-      "tbz x17, #3, 40f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x17, #2, 38f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 37f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x15], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x8, #3, 40f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x8, #2, 38f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 37f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x17, #1, 39f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x8, #1, 39f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x17, #2, 42f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 41f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x15], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x8, #2, 42f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 41f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 44f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 44f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x17, #1, 43f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x17, #0, 44f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x8, #1, 43f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "tbz x8, #0, 44f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "b 47f\n"
       "46:"  // Height 2: no accumulate
       "movi v8.4s, #0x0\n"
@@ -495,147 +495,147 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "cbnz x14, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "cbnz x15, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
       "50:"  // Height 2: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 53f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
       "ldr d6, [x16, #0x20]\n"
-      "add x12, x12, #0x10\n"
-      "ldr x10, [x16, #0x28]\n"
-      "add x28, x28, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      "sub x13, x13, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
-      "ldr x10, [x16, #0x48]\n"
-      "cmp x13, #0x20\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x58]\n"
-      "ldr x9, [x12, #0x8]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x58]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x78]\n"
-      "ldr x27, [x28, #0x8]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x98]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x98]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0xd8]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
+      "add x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "ldr d1, [x28, #0x0]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
+      "ldr d1, [x9, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x10, [x13, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x28, [x9, #0x8]\n"
+      "mov v0.d[1], x10\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
-      "sub x13, x13, #0x10\n"
-      "add x12, x12, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
+      ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       "ldr q6, [x16, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
@@ -671,17 +671,17 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x13, 58f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 58f\n"
+      "cmp x14, #0x4\n"
       "blt 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
@@ -693,22 +693,22 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 54b\n"
-      "cbz x13, 58f\n"
       "55:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 56f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "tbz x13, #0, 57f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
+      "cbz x14, 58f\n"
+      "tbz x14, #1, 56f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "tbz x14, #0, 57f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
       "b 57f\n"
       "56:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
       "57:"  // Height 2: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
@@ -720,203 +720,203 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "58:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "bge 67f\n"
-      "tbz x17, #3, 62f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "tbz x17, #2, 60f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 59f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
+      "tbz x8, #3, 62f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "tbz x8, #2, 60f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 59f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
       "b 66f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 66f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
       "b 66f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 61f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
+      "tbz x8, #1, 61f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
       "b 66f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 66f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
       "b 66f\n"
       "62:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 64f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "tbz x17, #1, 63f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
+      "tbz x8, #2, 64f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "tbz x8, #1, 63f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
       "b 66f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 66f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
+      "tbz x8, #0, 66f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
       "b 66f\n"
       "64:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 65f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "tbz x17, #0, 66f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
+      "tbz x8, #1, 65f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "tbz x8, #0, 66f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
       "b 66f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
       "66:"  // Height 2: Partial direct writeback: Done
       "b 68f\n"
       "67:"  // Height 2: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
       "68:"  // Height 2: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 36b\n"
       "b 206f\n"
       "69:"  // Height 3
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "70:"  // Height 3: Column loop
       "tbz %x[flags], #0, 80f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 79f\n"
-      "tbz x17, #3, 74f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x17, #2, 72f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 71f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x8, #3, 74f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x8, #2, 72f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 71f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 78f\n"
       "71:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 78f\n"
       "72:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x17, #1, 73f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x8, #1, 73f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 78f\n"
       "73:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 78f\n"
       "74:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x17, #2, 76f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 75f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x8, #2, 76f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 75f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 78f\n"
       "75:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 78f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 78f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 78f\n"
       "76:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x17, #1, 77f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "tbz x17, #0, 78f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "tbz x8, #1, 77f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "tbz x8, #0, 78f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 78f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
       "78:"  // Height 3: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 81f\n"
       "79:"  // Height 3: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
       "b 81f\n"
       "80:"  // Height 3: no accumulate
       "movi v8.4s, #0x0\n"
@@ -932,175 +932,175 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "81:"  // Height 3: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "82:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 83f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "cbnz x14, 84f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "cbnz x15, 84f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
       "b 84f\n"
       "83:"  // Height 3: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
       "84:"  // Height 3: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 87f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 86f\n"
       "85:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
-      "ldr x10, [x16, #0x28]\n"
-      "add x12, x12, #0x10\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "ldr x11, [x16, #0x38]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x40]\n"
-      "add x26, x26, #0x10\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x50]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x60]\n"
-      "cmp x13, #0x20\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0xf0]\n"
+      "mov v7.d[1], x11\n"
+      "add x9, x9, #0x10\n"
+      "add x27, x27, #0x10\n"
       "add x16, x16, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
-      "mov v0.d[1], x9\n"
-      "ldr d2, [x26, #0x0]\n"
-      "mov v1.d[1], x27\n"
-      "mov v2.d[1], x25\n"
+      "ldr d2, [x27, #0x0]\n"
+      "sub x14, x14, #0x10\n"
+      "ldr d7, [x16, #0x10]\n"
+      "cmp x14, #0x20\n"
+      "ldr x26, [x27, #0x8]\n"
+      "mov v6.d[1], x12\n"
+      "ldr x11, [x16, #0x18]\n"
+      "mov v0.d[1], x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
+      "mov v1.d[1], x28\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
+      "mov v2.d[1], x26\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
+      "mov v7.d[1], x11\n"
       "bge 85b\n"
       "86:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       "ldr q7, [x16, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
@@ -1151,18 +1151,18 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
       "87:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x13, 92f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 92f\n"
+      "cmp x14, #0x4\n"
       "blt 89f\n"
       "88:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
@@ -1178,25 +1178,25 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 88b\n"
-      "cbz x13, 92f\n"
       "89:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 90f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "tbz x13, #0, 91f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
+      "cbz x14, 92f\n"
+      "tbz x14, #1, 90f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "tbz x14, #0, 91f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
       "b 91f\n"
       "90:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
       "91:"  // Height 3: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
@@ -1212,246 +1212,246 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       "92:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 82b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "bge 101f\n"
-      "tbz x17, #3, 96f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x17, #2, 94f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 93f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x8, #3, 96f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x8, #2, 94f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 93f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 100f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 100f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 100f\n"
       "94:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 95f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x8, #1, 95f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 100f\n"
       "95:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 100f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 100f\n"
       "96:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 98f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x17, #1, 97f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x8, #2, 98f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x8, #1, 97f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 100f\n"
       "97:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 100f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x8, #0, 100f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 100f\n"
       "98:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 99f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x17, #0, 100f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x8, #1, 99f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x8, #0, 100f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 100f\n"
       "99:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "100:"  // Height 3: Partial direct writeback: Done
       "b 102f\n"
       "101:"  // Height 3: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "102:"  // Height 3: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 70b\n"
       "b 206f\n"
       "103:"  // Height 4
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "104:"  // Height 4: Column loop
       "tbz %x[flags], #0, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 113f\n"
-      "tbz x17, #3, 108f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x17, #2, 106f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 105f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x8, #3, 108f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x8, #2, 106f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 105f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 112f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 112f\n"
       "106:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x17, #1, 107f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x8, #1, 107f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 112f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 112f\n"
       "108:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x17, #2, 110f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 109f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x8, #2, 110f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 109f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 112f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 112f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 112f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 112f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x17, #1, 111f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x17, #0, 112f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x8, #1, 111f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x8, #0, 112f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 112f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "112:"  // Height 4: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 115f\n"
       "113:"  // Height 4: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "b 115f\n"
       "114:"  // Height 4: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1471,204 +1471,204 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "115:"  // Height 4: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "116:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 117f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x14, 118f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x15, 118f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
       "b 118f\n"
       "117:"  // Height 4: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
       "118:"  // Height 4: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 121f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 120f\n"
       "119:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
-      "mov v7.d[1], x11\n"
-      "ldr x11, [x16, #0x38]\n"
-      "add x12, x12, #0x10\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x68]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "add x26, x26, #0x10\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
-      ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
+      ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x70]\n"
-      "add x24, x24, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
-      "sub x13, x13, #0x10\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "cmp x14, #0x20\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
+      "ldr d3, [x25, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v7.d[1], x11\n"
       "bge 119b\n"
       "120:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
+      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
+      ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
@@ -1733,19 +1733,19 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
       "121:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x13, 126f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 126f\n"
+      "cmp x14, #0x4\n"
       "blt 123f\n"
       "122:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1765,28 +1765,28 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 122b\n"
-      "cbz x13, 126f\n"
       "123:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 124f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "tbz x13, #0, 125f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
+      "cbz x14, 126f\n"
+      "tbz x14, #1, 124f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "tbz x14, #0, 125f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
       "b 125f\n"
       "124:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
       "125:"  // Height 4: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -1806,289 +1806,289 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       "126:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 116b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "bge 135f\n"
-      "tbz x17, #3, 130f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "tbz x17, #2, 128f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 127f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "tbz x8, #3, 130f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "tbz x8, #2, 128f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 127f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 134f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 134f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 134f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 129f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "tbz x8, #1, 129f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 134f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 134f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 134f\n"
       "130:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 132f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "tbz x17, #1, 131f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "tbz x8, #2, 132f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "tbz x8, #1, 131f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 134f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 134f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "tbz x8, #0, 134f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 134f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 133f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x17, #0, 134f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "tbz x8, #1, 133f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x8, #0, 134f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 134f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "134:"  // Height 4: Partial direct writeback: Done
       "b 136f\n"
       "135:"  // Height 4: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
       "136:"  // Height 4: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 104b\n"
       "b 206f\n"
       "137:"  // Height 5
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
       "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "138:"  // Height 5: Column loop
       "tbz %x[flags], #0, 148f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 147f\n"
-      "tbz x17, #3, 142f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x17, #2, 140f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 139f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x8, #3, 142f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x8, #2, 140f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 139f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 146f\n"
       "139:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x17, #1, 141f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x8, #1, 141f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 146f\n"
       "141:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x17, #2, 144f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 143f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x8, #2, 144f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 143f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 146f\n"
       "143:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 146f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 146f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x17, #1, 145f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "tbz x17, #0, 146f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
+      "tbz x8, #1, 145f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "tbz x8, #0, 146f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
       "b 146f\n"
       "145:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
       "146:"  // Height 5: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 149f\n"
       "147:"  // Height 5: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
       "b 149f\n"
       "148:"  // Height 5: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2112,231 +2112,231 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "149:"  // Height 5: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "150:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 151f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "cbnz x14, 152f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "cbnz x15, 152f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
       "b 152f\n"
       "151:"  // Height 5: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
       "152:"  // Height 5: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 155f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 154f\n"
       "153:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      "add x28, x28, #0x10\n"
+      "ldr d6, [x16, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr d6, [x16, #0x20]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "mov v6.d[1], x10\n"
       "ldr d7, [x16, #0x30]\n"
-      "add x26, x26, #0x10\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "cmp x14, #0x20\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0xa8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8da  // udot v26.4s, v6.16b, v4.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
+      "ldr d4, [x23, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v7.d[1], x11\n"
       "bge 153b\n"
       "154:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
@@ -2417,20 +2417,20 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
       "155:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x13, 160f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 160f\n"
+      "cmp x14, #0x4\n"
       "blt 157f\n"
       "156:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2454,31 +2454,31 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 156b\n"
-      "cbz x13, 160f\n"
       "157:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 158f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "tbz x13, #0, 159f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "cbz x14, 160f\n"
+      "tbz x14, #1, 158f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "tbz x14, #0, 159f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 159f\n"
       "158:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "159:"  // Height 5: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -2502,335 +2502,335 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       "160:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 150b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "bge 169f\n"
-      "tbz x17, #3, 164f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x17, #2, 162f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 161f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x8, #3, 164f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x8, #2, 162f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 161f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 168f\n"
       "161:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 168f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 168f\n"
       "162:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 163f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x8, #1, 163f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 168f\n"
       "163:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 168f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 168f\n"
       "164:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 166f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x17, #1, 165f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x8, #2, 166f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x8, #1, 165f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 168f\n"
       "165:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 168f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x8, #0, 168f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 168f\n"
       "166:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 167f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x17, #0, 168f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x8, #1, 167f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x8, #0, 168f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 168f\n"
       "167:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "168:"  // Height 5: Partial direct writeback: Done
       "b 170f\n"
       "169:"  // Height 5: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "170:"  // Height 5: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 138b\n"
       "b 206f\n"
       "171:"  // Height 6
-      "ldr x17, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x15, %x[output_ptr]\n"
-      "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x8, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x17, %x[output_ptr]\n"
+      "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "172:"  // Height 6: Column loop
       "tbz %x[flags], #0, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 181f\n"
-      "tbz x17, #3, 176f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x15], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x17, #2, 174f\n"
-      "ld1 { v10.4s }, [x15], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 173f\n"
-      "ldr d11, [x15], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v11.s }[2], [x15]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x8, #3, 176f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x17], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x8, #2, 174f\n"
+      "ld1 { v10.4s }, [x17], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 173f\n"
+      "ldr d11, [x17], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d15, [x24], #0x8\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v11.s }[2], [x17]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 180f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s11, [x15, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s11, [x17, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 180f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x17, #1, 175f\n"
-      "ldr d10, [x15], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v10.s }[2], [x15]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x8, #1, 175f\n"
+      "ldr d10, [x17], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d14, [x24], #0x8\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v10.s }[2], [x17]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 180f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s10, [x15, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s10, [x17, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 180f\n"
       "176:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x17, #2, 178f\n"
-      "ld1 { v8.4s }, [x15], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 177f\n"
-      "ldr d9, [x15], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v9.s }[2], [x15]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x8, #2, 178f\n"
+      "ld1 { v8.4s }, [x17], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 177f\n"
+      "ldr d9, [x17], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d13, [x24], #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v9.s }[2], [x17]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 180f\n"
       "177:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x17, #0, 180f\n"
-      "ldr s9, [x15, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x8, #0, 180f\n"
+      "ldr s9, [x17, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 180f\n"
       "178:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x17, #1, 179f\n"
-      "ldr d8, [x15], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x17, #0, 180f\n"
-      "ld1 { v8.s }[2], [x15]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x8, #1, 179f\n"
+      "ldr d8, [x17], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x8, #0, 180f\n"
+      "ld1 { v8.s }[2], [x17]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 180f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x15, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s8, [x17, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s12, [x24, #0x0]\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "180:"  // Height 6: Partial accumulate: Done
-      "sub x15, x15, x24\n"
+      "sub x17, x17, x25\n"
       "b 183f\n"
       "181:"  // Height 6: full accumulate
-      "ldr q8, [x15, #0x0]\n"
-      "ldr q9, [x15, #0x10]\n"
-      "ldr q10, [x15, #0x20]\n"
-      "ldr q11, [x15, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q8, [x17, #0x0]\n"
+      "ldr q9, [x17, #0x10]\n"
+      "ldr q10, [x17, #0x20]\n"
+      "ldr q11, [x17, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "b 183f\n"
       "182:"  // Height 6: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2858,260 +2858,260 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "183:"  // Height 6: setup done
-      "mov x14, #0x0\n"
+      "mov x15, #0x0\n"
       "184:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w13, [x20, x14, LSL #0x2]\n"
+      "ldr w14, [x20, x15, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 185f\n"
-      "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x12, [x20, #0x0]\n"
-      "ldr x28, [x20, #0x8]\n"
-      "ldr x26, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x22, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x14, 186f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x12, x12, x19\n"
-      "add x28, x28, x19\n"
-      "add x26, x26, x19\n"
-      "add x24, x24, x19\n"
-      "add x22, x22, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x13, [x21, #0x0]\n"
+      "ldr x9, [x21, #0x8]\n"
+      "ldr x27, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x23, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x15, 186f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x13, x13, x20\n"
+      "add x9, x9, x20\n"
+      "add x27, x27, x20\n"
+      "add x25, x25, x20\n"
+      "add x23, x23, x20\n"
+      "add x21, x21, x20\n"
       "b 186f\n"
       "185:"  // Height 6: setup direct input
-      "mov x12, %x[input_ptr]\n"
-      "add x28, x12, x19\n"
-      "add x26, x28, x19\n"
-      "add x24, x26, x19\n"
-      "add x22, x24, x19\n"
-      "add x20, x22, x19\n"
+      "mov x13, %x[input_ptr]\n"
+      "add x9, x13, x20\n"
+      "add x27, x9, x20\n"
+      "add x25, x27, x20\n"
+      "add x23, x25, x20\n"
+      "add x21, x23, x20\n"
       "186:"  // Height 6: input setup done
-      "cmp x13, #0x10\n"
+      "cmp x14, #0x10\n"
       "blt 189f\n"
-      "ldr q0, [x12, #0x0]\n"
-      "ldr q1, [x28, #0x0]\n"
-      "cmp x13, #0x20\n"
-      "ldr q2, [x26, #0x0]\n"
-      "ldr q3, [x24, #0x0]\n"
-      "ldr q4, [x22, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q0, [x13, #0x0]\n"
+      "cmp x14, #0x20\n"
+      "ldr q1, [x9, #0x0]\n"
+      "ldr q2, [x27, #0x0]\n"
+      "ldr q3, [x25, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
       "ldr q6, [x16, #0x0]\n"
+      "ldr q7, [x16, #0x10]\n"
       "blt 188f\n"
       "187:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr d7, [x16, #0x10]\n"
+      "ldr x12, [x16, #0x28]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x18]\n"
+      "ldr x11, [x16, #0x38]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr x10, [x16, #0x28]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
-      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr d6, [x16, #0x20]\n"
+      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr x11, [x16, #0x38]\n"
+      "ldr x12, [x16, #0x48]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr x9, [x12, #0x8]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr x10, [x16, #0x48]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x16, #0x30]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x28, x28, #0x10\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
+      "ldr x11, [x16, #0x58]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "mov v7.d[1], x11\n"
+      "ldr x10, [x13, #0x8]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr x11, [x16, #0x58]\n"
+      "ldr x28, [x9, #0x8]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr x27, [x28, #0x8]\n"
+      "ldr x26, [x27, #0x8]\n"
       ".inst 0x6f85e0de  // udot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr d6, [x16, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "ldr x12, [x16, #0x68]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "mov v6.d[1], x10\n"
+      "ldr x24, [x25, #0x8]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr x10, [x16, #0x68]\n"
+      "ldr x22, [x23, #0x8]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr x25, [x26, #0x8]\n"
+      "ldr x20, [x21, #0x8]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
       "ldr d7, [x16, #0x50]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr x11, [x16, #0x78]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x78]\n"
+      "cmp x14, #0x20\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr x23, [x24, #0x8]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6fa5e0dc  // udot v28.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x16, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "add x22, x22, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr x12, [x16, #0x88]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0x88]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr x21, [x22, #0x8]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6fa5e0fd  // udot v29.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x16, #0x70]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "ldr x11, [x16, #0x98]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "mov v7.d[1], x11\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr x11, [x16, #0x98]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr x19, [x20, #0x8]\n"
       ".inst 0x6fa5e0de  // udot v30.4s, v6.16b, v5.4b[1]\n"
       "ldr d6, [x16, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "sub x13, x13, #0x10\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
-      "cmp x13, #0x20\n"
+      "ldr x12, [x16, #0xa8]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr x10, [x16, #0xa8]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0ff  // udot v31.4s, v7.16b, v5.4b[1]\n"
       "ldr d7, [x16, #0x90]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xb8]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8dc  // udot v28.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x16, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xc8]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xc8]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8fd  // udot v29.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x16, #0xb0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
+      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr x11, [x16, #0xd8]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8de  // udot v30.4s, v6.16b, v5.4b[2]\n"
       "ldr d6, [x16, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
+      "ldr x12, [x16, #0xe8]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr x10, [x16, #0xe8]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8ff  // udot v31.4s, v7.16b, v5.4b[2]\n"
       "ldr d7, [x16, #0xd0]\n"
+      "mov v7.d[1], x11\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
+      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr x11, [x16, #0xf8]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8dc  // udot v28.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x16, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
+      "mov v6.d[1], x12\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8fd  // udot v29.4s, v7.16b, v5.4b[3]\n"
       "ldr d7, [x16, #0xf0]\n"
-      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "mov v7.d[1], x11\n"
       "add x16, x16, #0x100\n"
+      ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
+      "ldr x12, [x16, #0x8]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr x10, [x16, #0x8]\n"
+      "ldr x11, [x16, #0x18]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
-      "mov v7.d[1], x11\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8da  // udot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8de  // udot v30.4s, v6.16b, v5.4b[3]\n"
       "ldr d6, [x16, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr d0, [x12, #0x0]\n"
+      "ldr d0, [x13, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr d1, [x28, #0x0]\n"
+      "ldr d1, [x9, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "mov v6.d[1], x10\n"
+      "ldr d2, [x27, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "mov v0.d[1], x9\n"
+      "ldr d3, [x25, #0x0]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
-      "mov v1.d[1], x27\n"
+      "ldr d4, [x23, #0x0]\n"
       ".inst 0x6fa5e8ff  // udot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr d2, [x26, #0x0]\n"
-      "ldr d3, [x24, #0x0]\n"
-      "ldr d4, [x22, #0x0]\n"
-      "mov v2.d[1], x25\n"
-      "ldr d5, [x20, #0x0]\n"
-      "mov v3.d[1], x23\n"
-      "mov v4.d[1], x21\n"
-      "mov v5.d[1], x19\n"
+      "ldr d5, [x21, #0x0]\n"
+      "ldr d7, [x16, #0x10]\n"
+      "mov v6.d[1], x12\n"
+      "mov v0.d[1], x10\n"
+      "mov v1.d[1], x28\n"
+      "mov v2.d[1], x26\n"
+      "mov v3.d[1], x24\n"
+      "mov v4.d[1], x22\n"
+      "mov v5.d[1], x20\n"
+      "mov v7.d[1], x11\n"
       "bge 187b\n"
       "188:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x16, #0x10]\n"
+      "add x13, x13, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "sub x13, x13, #0x10\n"
+      "add x9, x9, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x12, x12, #0x10\n"
+      "add x27, x27, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x12, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "add x28, x28, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x28, #0x80]\n"
-      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       "ldr q6, [x16, #0x20]\n"
+      ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "add x26, x26, #0x10\n"
+      "sub x14, x14, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x13, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "prfm pldl1keep, [x9, #0x80]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x27, #0x80]\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
       "ldr q7, [x16, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x22, x22, #0x10\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "add x20, x20, #0x10\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x6f85e0de  // udot v30.4s, v6.16b, v5.4b[0]\n"
       "ldr q6, [x16, #0x40]\n"
@@ -3206,21 +3206,21 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8ff  // udot v31.4s, v7.16b, v5.4b[3]\n"
       "189:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x13, 194f\n"
-      "cmp x13, #0x4\n"
+      "cbz x14, 194f\n"
+      "cmp x14, #0x4\n"
       "blt 191f\n"
       "190:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x12], #0x4\n"
-      "sub x13, x13, #0x4\n"
-      "ldr s1, [x28], #0x4\n"
-      "cmp x13, #0x4\n"
-      "ldr s2, [x26], #0x4\n"
-      "ldr s3, [x24], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
+      "ldr s0, [x13], #0x4\n"
+      "sub x14, x14, #0x4\n"
+      "ldr s1, [x9], #0x4\n"
+      "cmp x14, #0x4\n"
+      "ldr s2, [x27], #0x4\n"
+      "ldr s3, [x25], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3248,34 +3248,34 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 190b\n"
-      "cbz x13, 194f\n"
       "191:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x13, #1, 192f\n"
-      "ldr h0, [x12], #0x2\n"
-      "ldr h1, [x28], #0x2\n"
-      "ldr h2, [x26], #0x2\n"
-      "ldr h3, [x24], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "tbz x13, #0, 193f\n"
-      "ld1 { v0.b }[2], [x12]\n"
-      "ld1 { v1.b }[2], [x28]\n"
-      "ld1 { v2.b }[2], [x26]\n"
-      "ld1 { v3.b }[2], [x24]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "cbz x14, 194f\n"
+      "tbz x14, #1, 192f\n"
+      "ldr h0, [x13], #0x2\n"
+      "ldr h1, [x9], #0x2\n"
+      "ldr h2, [x27], #0x2\n"
+      "ldr h3, [x25], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "tbz x14, #0, 193f\n"
+      "ld1 { v0.b }[2], [x13]\n"
+      "ld1 { v1.b }[2], [x9]\n"
+      "ld1 { v2.b }[2], [x27]\n"
+      "ld1 { v3.b }[2], [x25]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 193f\n"
       "192:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x12, #0x0]\n"
-      "ldr b1, [x28, #0x0]\n"
-      "ldr b2, [x26, #0x0]\n"
-      "ldr b3, [x24, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b0, [x13, #0x0]\n"
+      "ldr b1, [x9, #0x0]\n"
+      "ldr b2, [x27, #0x0]\n"
+      "ldr b3, [x25, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "193:"  // Height 6: Multiply loop: Ragged operand read: Done
       "ldr q6, [x16, #0x0]\n"
-      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
+      "ldr q7, [x16, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
@@ -3303,195 +3303,195 @@ void a64_hybrid_u8u32_dot_6x16_a55 (
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
       "194:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x14, x14, #0x1\n"
-      "cmp x14, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x15, x15, #0x1\n"
+      "cmp x15, x20\n"
       "bne 184b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x15, #0x0]\n"
-      "cmp x17, #0x10\n"
-      "add x23, x15, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x17, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x8, #0x10\n"
+      "prfm pstl1keep, [x17, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "bge 203f\n"
-      "tbz x17, #3, 198f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v9.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "st1 { v29.4s }, [x19], #0x10\n"
-      "tbz x17, #2, 196f\n"
-      "st1 { v10.4s }, [x15], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v30.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 195f\n"
-      "str d11, [x15], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "str d31, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v11.s }[2], [x15]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
-      "st1 { v31.s }[2], [x19]\n"
+      "tbz x8, #3, 198f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v9.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v29.4s }, [x20], #0x10\n"
+      "tbz x8, #2, 196f\n"
+      "st1 { v10.4s }, [x17], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v30.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 195f\n"
+      "str d11, [x17], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "str d31, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v11.s }[2], [x17]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v31.s }[2], [x20]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x17, #0, 202f\n"
-      "str s11, [x15, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
-      "str s31, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s11, [x17, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
+      "str s31, [x20, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x17, #1, 197f\n"
-      "str d10, [x15], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "str d30, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v10.s }[2], [x15]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
-      "st1 { v30.s }[2], [x19]\n"
+      "tbz x8, #1, 197f\n"
+      "str d10, [x17], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "str d30, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v10.s }[2], [x17]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v30.s }[2], [x20]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x17, #0, 202f\n"
-      "str s10, [x15, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
-      "str s30, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s10, [x17, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
+      "str s30, [x20, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x17, #2, 200f\n"
-      "st1 { v8.4s }, [x15], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "tbz x17, #1, 199f\n"
-      "str d9, [x15], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "str d29, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v9.s }[2], [x15]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
-      "st1 { v29.s }[2], [x19]\n"
+      "tbz x8, #2, 200f\n"
+      "st1 { v8.4s }, [x17], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "tbz x8, #1, 199f\n"
+      "str d9, [x17], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "str d29, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v9.s }[2], [x17]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v29.s }[2], [x20]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x17, #0, 202f\n"
-      "str s9, [x15, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
-      "str s29, [x19, #0x0]\n"
+      "tbz x8, #0, 202f\n"
+      "str s9, [x17, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
+      "str s29, [x20, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x17, #1, 201f\n"
-      "str d8, [x15], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x17, #0, 202f\n"
-      "st1 { v8.s }[2], [x15]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
-      "st1 { v28.s }[2], [x19]\n"
+      "tbz x8, #1, 201f\n"
+      "str d8, [x17], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x8, #0, 202f\n"
+      "st1 { v8.s }[2], [x17]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v28.s }[2], [x20]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x15, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
-      "str s28, [x19, #0x0]\n"
+      "str s8, [x17, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
+      "str s28, [x20, #0x0]\n"
       "202:"  // Height 6: Partial direct writeback: Done
       "b 204f\n"
       "203:"  // Height 6: Full writeback
-      "str q8, [x15, #0x0]\n"
-      "str q9, [x15, #0x10]\n"
-      "str q10, [x15, #0x20]\n"
-      "str q11, [x15, #0x30]\n"
-      "add x15, x15, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
-      "str q28, [x19, #0x0]\n"
-      "str q29, [x19, #0x10]\n"
-      "str q30, [x19, #0x20]\n"
-      "str q31, [x19, #0x30]\n"
+      "str q8, [x17, #0x0]\n"
+      "str q9, [x17, #0x10]\n"
+      "str q10, [x17, #0x20]\n"
+      "str q11, [x17, #0x30]\n"
+      "add x17, x17, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
+      "str q28, [x20, #0x0]\n"
+      "str q29, [x20, #0x10]\n"
+      "str q30, [x20, #0x20]\n"
+      "str q31, [x20, #0x30]\n"
       "204:"  // Height 6: Writeback done
-      "subs x17, x17, #0x10\n"
+      "subs x8, x8, #0x10\n"
       "bgt 172b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 206f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 205f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "205:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "206:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp
index ab0c88a3b2..38131cfd4b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_u8u32_dot_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 69f\n"
       "beq 35f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 12f\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
       "bge 11f\n"
-      "tbz x10, #3, 6f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 4f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 6f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 4f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 3f\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x10, #1, 5f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 5f\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x10, #2, 8f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 7f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 8f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 7f\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s9, [x28, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s9, [x9, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x10, #1, 9f\n"
-      "ldr d8, [x28], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 9f\n"
+      "ldr d8, [x9], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v8.s }[2], [x9]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s8, [x9, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 13f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
       "b 13f\n"
       "12:"  // Height 1: no accumulate
       "movi v8.4s, #0x0\n"
@@ -161,295 +161,295 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
       "13:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "14:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 15f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 16f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 16f\n"
       "15:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "16:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 19f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 18f\n"
       "17:"  // Height 1: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "add x10, x10, #0x100\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 17b\n"
       "18:"  // Height 1: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "19:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 24f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 24f\n"
+      "cmp x27, #0x4\n"
       "blt 21f\n"
       "20:"  // Height 1: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "cmp x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "ldr q7, [x10, #0x10]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "cmp x27, #0x4\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "add x9, x9, #0x40\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x10, x10, #0x40\n"
       "bge 20b\n"
-      "cbz x26, 24f\n"
       "21:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 22f\n"
-      "ldr h0, [x25], #0x2\n"
-      "tbz x26, #0, 23f\n"
-      "ld1 { v0.b }[2], [x25]\n"
+      "cbz x27, 24f\n"
+      "tbz x27, #1, 22f\n"
+      "ldr h0, [x26], #0x2\n"
+      "tbz x27, #0, 23f\n"
+      "ld1 { v0.b }[2], [x26]\n"
       "b 23f\n"
       "22:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q6, [x10, #0x20]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "add x10, x10, #0x40\n"
       "24:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 14b\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "bge 33f\n"
-      "tbz x10, #3, 28f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 26f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 25f\n"
-      "str d11, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 28f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 26f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 25f\n"
+      "str d11, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 32f\n"
       "25:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 32f\n"
-      "str s11, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s11, [x9, #0x0]\n"
       "b 32f\n"
       "26:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 27f\n"
-      "str d10, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 27f\n"
+      "str d10, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 32f\n"
       "27:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 32f\n"
-      "str s10, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s10, [x9, #0x0]\n"
       "b 32f\n"
       "28:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 30f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 29f\n"
-      "str d9, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 30f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 29f\n"
+      "str d9, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 32f\n"
       "29:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 32f\n"
-      "str s9, [x28, #0x0]\n"
+      "tbz x11, #0, 32f\n"
+      "str s9, [x9, #0x0]\n"
       "b 32f\n"
       "30:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 31f\n"
-      "str d8, [x28], #0x8\n"
-      "tbz x10, #0, 32f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 31f\n"
+      "str d8, [x9], #0x8\n"
+      "tbz x11, #0, 32f\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 32f\n"
       "31:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "32:"  // Height 1: Partial direct writeback: Done
       "b 34f\n"
       "33:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "34:"  // Height 1: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 206f\n"
       "35:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "36:"  // Height 2: Column loop
       "tbz %x[flags], #0, 46f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
+      "add x24, x9, x20, LSL #2\n"
       "bge 45f\n"
-      "tbz x10, #3, 40f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 38f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 37f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 40f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 38f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 37f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 44f\n"
       "37:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 44f\n"
       "38:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x10, #1, 39f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 39f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 44f\n"
       "39:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 44f\n"
       "40:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x10, #2, 42f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 41f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 42f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 41f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 44f\n"
       "41:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 44f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 44f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 44f\n"
       "42:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x10, #1, 43f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 44f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 43f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 44f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 44f\n"
       "43:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
       "44:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 47f\n"
       "45:"  // Height 2: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "b 47f\n"
       "46:"  // Height 2: no accumulate
       "movi v8.4s, #0x0\n"
@@ -461,392 +461,392 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "47:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "48:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 49f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 50f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 50f\n"
       "49:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "50:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 53f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 52f\n"
       "51:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 51b\n"
       "52:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
       "53:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 58f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 58f\n"
+      "cmp x27, #0x4\n"
       "blt 55f\n"
       "54:"  // Height 2: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "bge 54b\n"
-      "cbz x26, 58f\n"
       "55:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 56f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "tbz x26, #0, 57f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
+      "cbz x27, 58f\n"
+      "tbz x27, #1, 56f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "tbz x27, #0, 57f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
       "b 57f\n"
       "56:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
       "57:"  // Height 2: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       "58:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 48b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "bge 67f\n"
-      "tbz x10, #3, 62f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 60f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 59f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 62f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 60f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 59f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
       "b 66f\n"
       "59:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 66f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
       "b 66f\n"
       "60:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 61f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 61f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
       "b 66f\n"
       "61:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 66f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
       "b 66f\n"
       "62:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 64f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 63f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 64f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 63f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
       "b 66f\n"
       "63:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 66f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
+      "tbz x11, #0, 66f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
       "b 66f\n"
       "64:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 65f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "tbz x10, #0, 66f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 65f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "tbz x11, #0, 66f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
       "b 66f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
       "66:"  // Height 2: Partial direct writeback: Done
       "b 68f\n"
       "67:"  // Height 2: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
       "68:"  // Height 2: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 36b\n"
       "b 206f\n"
       "69:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "70:"  // Height 3: Column loop
       "tbz %x[flags], #0, 80f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 79f\n"
-      "tbz x10, #3, 74f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 72f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 71f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 74f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 72f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 71f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 78f\n"
       "71:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 78f\n"
       "72:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x10, #1, 73f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 73f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 78f\n"
       "73:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 78f\n"
       "74:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x10, #2, 76f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 75f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 76f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 75f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 78f\n"
       "75:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 78f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 78f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
       "b 78f\n"
       "76:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x10, #1, 77f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "tbz x10, #0, 78f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 77f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "tbz x11, #0, 78f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
       "b 78f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
       "78:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 81f\n"
       "79:"  // Height 3: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
       "b 81f\n"
       "80:"  // Height 3: no accumulate
       "movi v8.4s, #0x0\n"
@@ -862,180 +862,180 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
       "81:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "82:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 83f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 84f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 84f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 84f\n"
       "83:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "84:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 87f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 86f\n"
       "85:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 85b\n"
       "86:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
@@ -1043,26 +1043,26 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
       "87:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 92f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 92f\n"
+      "cmp x27, #0x4\n"
       "blt 89f\n"
       "88:"  // Height 3: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1070,33 +1070,33 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       "bge 88b\n"
-      "cbz x26, 92f\n"
       "89:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 90f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "tbz x26, #0, 91f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
+      "cbz x27, 92f\n"
+      "tbz x27, #1, 90f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "tbz x27, #0, 91f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
       "b 91f\n"
       "90:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
       "91:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1104,246 +1104,246 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       "92:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 82b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "bge 101f\n"
-      "tbz x10, #3, 96f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 94f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 93f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 96f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 94f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 93f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 100f\n"
       "93:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 100f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 100f\n"
       "94:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 95f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 95f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 100f\n"
       "95:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 100f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 100f\n"
       "96:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 98f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 97f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 98f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 97f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 100f\n"
       "97:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 100f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x11, #0, 100f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 100f\n"
       "98:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 99f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x10, #0, 100f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 99f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x11, #0, 100f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 100f\n"
       "99:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "100:"  // Height 3: Partial direct writeback: Done
       "b 102f\n"
       "101:"  // Height 3: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "102:"  // Height 3: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 70b\n"
       "b 206f\n"
       "103:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "104:"  // Height 4: Column loop
       "tbz %x[flags], #0, 114f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 113f\n"
-      "tbz x10, #3, 108f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 106f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 105f\n"
-      "mov x24, #0x38\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 108f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 106f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 105f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 112f\n"
       "105:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 112f\n"
       "106:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x10, #1, 107f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 107f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 112f\n"
       "107:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 112f\n"
       "108:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x10, #2, 110f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 109f\n"
-      "mov x24, #0x18\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 110f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 109f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 112f\n"
       "109:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 112f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 112f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 112f\n"
       "110:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x10, #1, 111f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x10, #0, 112f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 111f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x11, #0, 112f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 112f\n"
       "111:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "112:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 115f\n"
       "113:"  // Height 4: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "b 115f\n"
       "114:"  // Height 4: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1363,219 +1363,219 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "115:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "116:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 117f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 118f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 118f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 118f\n"
       "117:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "118:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 121f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 120f\n"
       "119:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 119b\n"
       "120:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "add x23, x23, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
@@ -1585,29 +1585,29 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
       "121:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 126f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 126f\n"
+      "cmp x27, #0x4\n"
       "blt 123f\n"
       "122:"  // Height 4: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1617,38 +1617,38 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       "bge 122b\n"
-      "cbz x26, 126f\n"
       "123:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 124f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "tbz x26, #0, 125f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
+      "cbz x27, 126f\n"
+      "tbz x27, #1, 124f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "tbz x27, #0, 125f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
       "b 125f\n"
       "124:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
       "125:"  // Height 4: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -1658,289 +1658,289 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       "126:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 116b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
       "bge 135f\n"
-      "tbz x10, #3, 130f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 128f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 127f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 130f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 128f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 127f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
       "b 134f\n"
       "127:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 134f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
       "b 134f\n"
       "128:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 129f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 129f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
       "b 134f\n"
       "129:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 134f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
       "b 134f\n"
       "130:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 132f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 131f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 132f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 131f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
       "b 134f\n"
       "131:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 134f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
+      "tbz x11, #0, 134f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
       "b 134f\n"
       "132:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 133f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "tbz x10, #0, 134f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 133f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "tbz x11, #0, 134f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
       "b 134f\n"
       "133:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
       "134:"  // Height 4: Partial direct writeback: Done
       "b 136f\n"
       "135:"  // Height 4: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
       "136:"  // Height 4: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 104b\n"
       "b 206f\n"
       "137:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "138:"  // Height 5: Column loop
       "tbz %x[flags], #0, 148f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 147f\n"
-      "tbz x10, #3, 142f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 140f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 139f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 142f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 140f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 139f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 146f\n"
       "139:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x10, #1, 141f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 141f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 146f\n"
       "141:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x10, #2, 144f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 143f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 144f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 143f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 146f\n"
       "143:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 146f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 146f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x10, #1, 145f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 145f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
       "b 146f\n"
       "145:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
       "146:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 149f\n"
       "147:"  // Height 5: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
       "b 149f\n"
       "148:"  // Height 5: no accumulate
       "movi v8.4s, #0x0\n"
@@ -1964,258 +1964,258 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v26.4s, #0x0\n"
       "movi v27.4s, #0x0\n"
       "149:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "150:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 151f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 152f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 152f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 152f\n"
       "151:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "152:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 155f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 154f\n"
       "153:"  // Height 5: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
+      "add x22, x22, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8da  // udot v26.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 153b\n"
       "154:"  // Height 5: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2227,32 +2227,32 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
       "155:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 160f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 160f\n"
+      "cmp x27, #0x4\n"
       "blt 157f\n"
       "156:"  // Height 5: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2264,43 +2264,43 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       "bge 156b\n"
-      "cbz x26, 160f\n"
       "157:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 158f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "tbz x26, #0, 159f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
+      "cbz x27, 160f\n"
+      "tbz x27, #1, 158f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "tbz x27, #0, 159f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
       "b 159f\n"
       "158:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
       "159:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -2312,335 +2312,335 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       "160:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 150b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "cmp x11, #0x10\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "bge 169f\n"
-      "tbz x10, #3, 164f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 162f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 161f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 164f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 162f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 161f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 168f\n"
       "161:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 168f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 168f\n"
       "162:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 163f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 163f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 168f\n"
       "163:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 168f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 168f\n"
       "164:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 166f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 165f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 166f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 165f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 168f\n"
       "165:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 168f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x11, #0, 168f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 168f\n"
       "166:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 167f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x10, #0, 168f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 167f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x11, #0, 168f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 168f\n"
       "167:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "168:"  // Height 5: Partial direct writeback: Done
       "b 170f\n"
       "169:"  // Height 5: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "170:"  // Height 5: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 138b\n"
       "b 206f\n"
       "171:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "172:"  // Height 6: Column loop
       "tbz %x[flags], #0, 182f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 181f\n"
-      "tbz x10, #3, 176f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 174f\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 173f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 176f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 174f\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 173f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 180f\n"
       "173:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 180f\n"
       "174:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x10, #1, 175f\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 175f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 180f\n"
       "175:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 180f\n"
       "176:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x10, #2, 178f\n"
-      "ld1 { v8.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v16.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v24.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 177f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 178f\n"
+      "ld1 { v8.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v16.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v24.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 177f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 180f\n"
       "177:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 180f\n"
-      "ldr s9, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 180f\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 180f\n"
       "178:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x10, #1, 179f\n"
-      "ldr d8, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d16, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d24, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x10, #0, 180f\n"
-      "ld1 { v8.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v16.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v24.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 179f\n"
+      "ldr d8, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d16, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d24, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x11, #0, 180f\n"
+      "ld1 { v8.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v16.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v24.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 180f\n"
       "179:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s8, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s16, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s24, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s8, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s16, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s24, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "180:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 183f\n"
       "181:"  // Height 6: full accumulate
-      "ldr q8, [x28, #0x0]\n"
-      "ldr q9, [x28, #0x10]\n"
-      "ldr q10, [x28, #0x20]\n"
-      "ldr q11, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q16, [x22, #0x0]\n"
-      "ldr q17, [x22, #0x10]\n"
-      "ldr q18, [x22, #0x20]\n"
-      "ldr q19, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q24, [x20, #0x0]\n"
-      "ldr q25, [x20, #0x10]\n"
-      "ldr q26, [x20, #0x20]\n"
-      "ldr q27, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q8, [x9, #0x0]\n"
+      "ldr q9, [x9, #0x10]\n"
+      "ldr q10, [x9, #0x20]\n"
+      "ldr q11, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q16, [x23, #0x0]\n"
+      "ldr q17, [x23, #0x10]\n"
+      "ldr q18, [x23, #0x20]\n"
+      "ldr q19, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q24, [x21, #0x0]\n"
+      "ldr q25, [x21, #0x10]\n"
+      "ldr q26, [x21, #0x20]\n"
+      "ldr q27, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "b 183f\n"
       "182:"  // Height 6: no accumulate
       "movi v8.4s, #0x0\n"
@@ -2668,297 +2668,297 @@ void a64_hybrid_u8u32_dot_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "183:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "184:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 185f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 186f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 186f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 186f\n"
       "185:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "186:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 189f\n"
-      "ldr q0, [x25, #0x0]\n"
-      "ldr q1, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
-      "ldr q2, [x23, #0x0]\n"
-      "ldr q3, [x22, #0x0]\n"
-      "ldr q4, [x21, #0x0]\n"
-      "ldr q5, [x20, #0x0]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q2, [x24, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "blt 188f\n"
       "187:"  // Height 6: Multiply loop: Main loop head
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x24, x24, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x20, x20, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
+      "cmp x27, #0x20\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x6f85e0de  // udot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0dc  // udot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0fd  // udot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0de  // udot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0ff  // udot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8dc  // udot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8fd  // udot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8de  // udot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8ff  // udot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8dc  // udot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8fd  // udot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d6  // udot v22.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8da  // udot v26.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8de  // udot v30.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
       ".inst 0x6fa0e8eb  // udot v11.4s, v7.16b, v0.4b[3]\n"
-      "ldr q0, [x25, #0x0]\n"
+      "ldr q0, [x26, #0x0]\n"
       ".inst 0x6fa1e8ef  // udot v15.4s, v7.16b, v1.4b[3]\n"
-      "ldr q1, [x24, #0x0]\n"
+      "ldr q1, [x25, #0x0]\n"
       ".inst 0x6fa2e8f3  // udot v19.4s, v7.16b, v2.4b[3]\n"
-      "ldr q2, [x23, #0x0]\n"
+      "ldr q2, [x24, #0x0]\n"
       ".inst 0x6fa3e8f7  // udot v23.4s, v7.16b, v3.4b[3]\n"
-      "ldr q3, [x22, #0x0]\n"
+      "ldr q3, [x23, #0x0]\n"
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
-      "ldr q4, [x21, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x6fa5e8ff  // udot v31.4s, v7.16b, v5.4b[3]\n"
-      "ldr q5, [x20, #0x0]\n"
+      "ldr q5, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       "bge 187b\n"
       "188:"  // Height 6: Multiply loop: Single iteration only
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q6, [x10, #0x20]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d6  // udot v22.4s, v6.16b, v3.4b[0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6f84e0da  // udot v26.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x6f85e0de  // udot v30.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6f80e0eb  // udot v11.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ef  // udot v15.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f3  // udot v19.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f7  // udot v23.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6fa0e0c8  // udot v8.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0cc  // udot v12.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d0  // udot v16.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d4  // udot v20.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0d8  // udot v24.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0dc  // udot v28.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6fa0e0e9  // udot v9.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ed  // udot v13.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f1  // udot v17.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f5  // udot v21.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0f9  // udot v25.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0fd  // udot v29.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x70]\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6fa0e0ca  // udot v10.4s, v6.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ce  // udot v14.4s, v6.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0d2  // udot v18.4s, v6.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0d6  // udot v22.4s, v6.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0da  // udot v26.4s, v6.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0de  // udot v30.4s, v6.16b, v5.4b[1]\n"
-      "ldr q6, [x9, #0x80]\n"
+      "ldr q6, [x10, #0x80]\n"
       ".inst 0x6fa0e0eb  // udot v11.4s, v7.16b, v0.4b[1]\n"
       ".inst 0x6fa1e0ef  // udot v15.4s, v7.16b, v1.4b[1]\n"
       ".inst 0x6fa2e0f3  // udot v19.4s, v7.16b, v2.4b[1]\n"
       ".inst 0x6fa3e0f7  // udot v23.4s, v7.16b, v3.4b[1]\n"
       ".inst 0x6fa4e0fb  // udot v27.4s, v7.16b, v4.4b[1]\n"
       ".inst 0x6fa5e0ff  // udot v31.4s, v7.16b, v5.4b[1]\n"
-      "ldr q7, [x9, #0x90]\n"
+      "ldr q7, [x10, #0x90]\n"
       ".inst 0x6f80e8c8  // udot v8.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8cc  // udot v12.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d0  // udot v16.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d4  // udot v20.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8d8  // udot v24.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8dc  // udot v28.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xa0]\n"
+      "ldr q6, [x10, #0xa0]\n"
       ".inst 0x6f80e8e9  // udot v9.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ed  // udot v13.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f1  // udot v17.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f5  // udot v21.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8f9  // udot v25.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8fd  // udot v29.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xb0]\n"
+      "ldr q7, [x10, #0xb0]\n"
       ".inst 0x6f80e8ca  // udot v10.4s, v6.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ce  // udot v14.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d2  // udot v18.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8d6  // udot v22.4s, v6.16b, v3.4b[2]\n"
       ".inst 0x6f84e8da  // udot v26.4s, v6.16b, v4.4b[2]\n"
       ".inst 0x6f85e8de  // udot v30.4s, v6.16b, v5.4b[2]\n"
-      "ldr q6, [x9, #0xc0]\n"
+      "ldr q6, [x10, #0xc0]\n"
       ".inst 0x6f80e8eb  // udot v11.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8ef  // udot v15.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8f3  // udot v19.4s, v7.16b, v2.4b[2]\n"
       ".inst 0x6f83e8f7  // udot v23.4s, v7.16b, v3.4b[2]\n"
       ".inst 0x6f84e8fb  // udot v27.4s, v7.16b, v4.4b[2]\n"
       ".inst 0x6f85e8ff  // udot v31.4s, v7.16b, v5.4b[2]\n"
-      "ldr q7, [x9, #0xd0]\n"
+      "ldr q7, [x10, #0xd0]\n"
       ".inst 0x6fa0e8c8  // udot v8.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8cc  // udot v12.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d0  // udot v16.4s, v6.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8d4  // udot v20.4s, v6.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8d8  // udot v24.4s, v6.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8dc  // udot v28.4s, v6.16b, v5.4b[3]\n"
-      "ldr q6, [x9, #0xe0]\n"
+      "ldr q6, [x10, #0xe0]\n"
       ".inst 0x6fa0e8e9  // udot v9.4s, v7.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ed  // udot v13.4s, v7.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8f1  // udot v17.4s, v7.16b, v2.4b[3]\n"
       ".inst 0x6fa3e8f5  // udot v21.4s, v7.16b, v3.4b[3]\n"
       ".inst 0x6fa4e8f9  // udot v25.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8fd  // udot v29.4s, v7.16b, v5.4b[3]\n"
-      "ldr q7, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q7, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e8ca  // udot v10.4s, v6.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8ce  // udot v14.4s, v6.16b, v1.4b[3]\n"
       ".inst 0x6fa2e8d2  // udot v18.4s, v6.16b, v2.4b[3]\n"
@@ -2972,35 +2972,35 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6fa4e8fb  // udot v27.4s, v7.16b, v4.4b[3]\n"
       ".inst 0x6fa5e8ff  // udot v31.4s, v7.16b, v5.4b[3]\n"
       "189:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 194f\n"
-      "cmp x26, #0x4\n"
+      "cbz x27, 194f\n"
+      "cmp x27, #0x4\n"
       "blt 191f\n"
       "190:"  // Height 6: Multiply loop: Odd block loop
-      "ldr s0, [x25], #0x4\n"
-      "sub x26, x26, #0x4\n"
-      "ldr s1, [x24], #0x4\n"
-      "cmp x26, #0x4\n"
-      "ldr s2, [x23], #0x4\n"
-      "ldr s3, [x22], #0x4\n"
-      "ldr s4, [x21], #0x4\n"
-      "ldr s5, [x20], #0x4\n"
-      "ldr q6, [x9, #0x0]\n"
+      "ldr s0, [x26], #0x4\n"
+      "ldr s1, [x25], #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
+      "ldr s2, [x24], #0x4\n"
+      "ldr s3, [x23], #0x4\n"
+      "ldr s4, [x22], #0x4\n"
+      "ldr s5, [x21], #0x4\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3014,48 +3014,48 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
       "bge 190b\n"
-      "cbz x26, 194f\n"
       "191:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #1, 192f\n"
-      "ldr h0, [x25], #0x2\n"
-      "ldr h1, [x24], #0x2\n"
-      "ldr h2, [x23], #0x2\n"
-      "ldr h3, [x22], #0x2\n"
-      "ldr h4, [x21], #0x2\n"
-      "ldr h5, [x20], #0x2\n"
-      "tbz x26, #0, 193f\n"
-      "ld1 { v0.b }[2], [x25]\n"
-      "ld1 { v1.b }[2], [x24]\n"
-      "ld1 { v2.b }[2], [x23]\n"
-      "ld1 { v3.b }[2], [x22]\n"
-      "ld1 { v4.b }[2], [x21]\n"
-      "ld1 { v5.b }[2], [x20]\n"
+      "cbz x27, 194f\n"
+      "tbz x27, #1, 192f\n"
+      "ldr h0, [x26], #0x2\n"
+      "ldr h1, [x25], #0x2\n"
+      "ldr h2, [x24], #0x2\n"
+      "ldr h3, [x23], #0x2\n"
+      "ldr h4, [x22], #0x2\n"
+      "ldr h5, [x21], #0x2\n"
+      "tbz x27, #0, 193f\n"
+      "ld1 { v0.b }[2], [x26]\n"
+      "ld1 { v1.b }[2], [x25]\n"
+      "ld1 { v2.b }[2], [x24]\n"
+      "ld1 { v3.b }[2], [x23]\n"
+      "ld1 { v4.b }[2], [x22]\n"
+      "ld1 { v5.b }[2], [x21]\n"
       "b 193f\n"
       "192:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b0, [x25, #0x0]\n"
-      "ldr b1, [x24, #0x0]\n"
-      "ldr b2, [x23, #0x0]\n"
-      "ldr b3, [x22, #0x0]\n"
-      "ldr b4, [x21, #0x0]\n"
-      "ldr b5, [x20, #0x0]\n"
+      "ldr b0, [x26, #0x0]\n"
+      "ldr b1, [x25, #0x0]\n"
+      "ldr b2, [x24, #0x0]\n"
+      "ldr b3, [x23, #0x0]\n"
+      "ldr b4, [x22, #0x0]\n"
+      "ldr b5, [x21, #0x0]\n"
       "193:"  // Height 6: Multiply loop: Ragged operand read: Done
-      "ldr q6, [x9, #0x0]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6f80e0c8  // udot v8.4s, v6.16b, v0.4b[0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6f81e0cc  // udot v12.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d0  // udot v16.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0d4  // udot v20.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f84e0d8  // udot v24.4s, v6.16b, v4.4b[0]\n"
       ".inst 0x6f85e0dc  // udot v28.4s, v6.16b, v5.4b[0]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6f80e0e9  // udot v9.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ed  // udot v13.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f82e0f1  // udot v17.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0f5  // udot v21.4s, v7.16b, v3.4b[0]\n"
       ".inst 0x6f84e0f9  // udot v25.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0fd  // udot v29.4s, v7.16b, v5.4b[0]\n"
-      "ldr q7, [x9, #0x30]\n"
-      "add x9, x9, #0x40\n"
+      "ldr q7, [x10, #0x30]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0ca  // udot v10.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0ce  // udot v14.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0d2  // udot v18.4s, v6.16b, v2.4b[0]\n"
@@ -3069,195 +3069,195 @@ void a64_hybrid_u8u32_dot_6x16 (
       ".inst 0x6f84e0fb  // udot v27.4s, v7.16b, v4.4b[0]\n"
       ".inst 0x6f85e0ff  // udot v31.4s, v7.16b, v5.4b[0]\n"
       "194:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 184b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "prfm pstl1keep, [x23, #0x0]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
       "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
       "bge 203f\n"
-      "tbz x10, #3, 198f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v13.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v21.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "st1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 196f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "st1 { v14.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "st1 { v22.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "st1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 195f\n"
-      "str d11, [x28], #0x8\n"
-      "str d15, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "str d23, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "str d31, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v11.s }[2], [x28]\n"
-      "st1 { v15.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
-      "st1 { v23.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
-      "st1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 198f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v13.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v21.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "st1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 196f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "st1 { v14.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "st1 { v22.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "st1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 195f\n"
+      "str d11, [x9], #0x8\n"
+      "str d15, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "str d23, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "str d31, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v11.s }[2], [x9]\n"
+      "st1 { v15.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
+      "st1 { v23.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
+      "st1 { v31.s }[2], [x20]\n"
       "b 202f\n"
       "195:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 202f\n"
-      "str s11, [x28, #0x0]\n"
-      "str s15, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
-      "str s23, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
-      "str s31, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s11, [x9, #0x0]\n"
+      "str s15, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
+      "str s23, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
+      "str s31, [x20, #0x0]\n"
       "b 202f\n"
       "196:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 197f\n"
-      "str d10, [x28], #0x8\n"
-      "str d14, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "str d22, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "str d30, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v10.s }[2], [x28]\n"
-      "st1 { v14.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
-      "st1 { v22.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
-      "st1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 197f\n"
+      "str d10, [x9], #0x8\n"
+      "str d14, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "str d22, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "str d30, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v10.s }[2], [x9]\n"
+      "st1 { v14.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
+      "st1 { v22.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
+      "st1 { v30.s }[2], [x20]\n"
       "b 202f\n"
       "197:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 202f\n"
-      "str s10, [x28, #0x0]\n"
-      "str s14, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
-      "str s22, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
-      "str s30, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s10, [x9, #0x0]\n"
+      "str s14, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
+      "str s22, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
+      "str s30, [x20, #0x0]\n"
       "b 202f\n"
       "198:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 200f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 199f\n"
-      "str d9, [x28], #0x8\n"
-      "str d13, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "str d21, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "str d29, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v9.s }[2], [x28]\n"
-      "st1 { v13.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
-      "st1 { v21.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
-      "st1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 200f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 199f\n"
+      "str d9, [x9], #0x8\n"
+      "str d13, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "str d21, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "str d29, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v9.s }[2], [x9]\n"
+      "st1 { v13.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
+      "st1 { v21.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
+      "st1 { v29.s }[2], [x20]\n"
       "b 202f\n"
       "199:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 202f\n"
-      "str s9, [x28, #0x0]\n"
-      "str s13, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
-      "str s21, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
-      "str s29, [x19, #0x0]\n"
+      "tbz x11, #0, 202f\n"
+      "str s9, [x9, #0x0]\n"
+      "str s13, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
+      "str s21, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
+      "str s29, [x20, #0x0]\n"
       "b 202f\n"
       "200:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 201f\n"
-      "str d8, [x28], #0x8\n"
-      "str d12, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "str d20, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "str d28, [x19], #0x8\n"
-      "tbz x10, #0, 202f\n"
-      "st1 { v8.s }[2], [x28]\n"
-      "st1 { v12.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
-      "st1 { v20.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
-      "st1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 201f\n"
+      "str d8, [x9], #0x8\n"
+      "str d12, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "str d20, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "str d28, [x20], #0x8\n"
+      "tbz x11, #0, 202f\n"
+      "st1 { v8.s }[2], [x9]\n"
+      "st1 { v12.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
+      "st1 { v20.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
+      "st1 { v28.s }[2], [x20]\n"
       "b 202f\n"
       "201:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
-      "str s12, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
-      "str s20, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
-      "str s28, [x19, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
+      "str s12, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
+      "str s20, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
+      "str s28, [x20, #0x0]\n"
       "202:"  // Height 6: Partial direct writeback: Done
       "b 204f\n"
       "203:"  // Height 6: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q12, [x23, #0x0]\n"
-      "str q13, [x23, #0x10]\n"
-      "str q14, [x23, #0x20]\n"
-      "str q15, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
-      "str q20, [x21, #0x0]\n"
-      "str q21, [x21, #0x10]\n"
-      "str q22, [x21, #0x20]\n"
-      "str q23, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
-      "str q28, [x19, #0x0]\n"
-      "str q29, [x19, #0x10]\n"
-      "str q30, [x19, #0x20]\n"
-      "str q31, [x19, #0x30]\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q12, [x24, #0x0]\n"
+      "str q13, [x24, #0x10]\n"
+      "str q14, [x24, #0x20]\n"
+      "str q15, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
+      "str q20, [x22, #0x0]\n"
+      "str q21, [x22, #0x10]\n"
+      "str q22, [x22, #0x20]\n"
+      "str q23, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
+      "str q28, [x20, #0x0]\n"
+      "str q29, [x20, #0x10]\n"
+      "str q30, [x20, #0x20]\n"
+      "str q31, [x20, #0x30]\n"
       "204:"  // Height 6: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 172b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 206f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 205f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "205:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "206:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp
index fabb3f3efd..dd0c46e4dc 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -87,73 +87,73 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "cmp %x[M], #0x2\n"
       "bgt 75f\n"
       "beq 38f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
       "tbz %x[flags], #0, 13f\n"
-      "cmp x10, #0x10\n"
+      "cmp x11, #0x10\n"
       "bge 11f\n"
-      "tbz x10, #3, 6f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 4f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 3f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v16.s }[2], [x28]\n"
+      "tbz x11, #3, 6f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 4f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 3f\n"
+      "ldr d16, [x9], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v16.s }[2], [x9]\n"
       "b 10f\n"
       "3:"  // Height 1: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s16, [x28, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s16, [x9, #0x0]\n"
       "b 10f\n"
       "4:"  // Height 1: Partial accumulate: partial_2_8
-      "tbz x10, #1, 5f\n"
-      "ldr d11, [x28], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v11.s }[2], [x28]\n"
+      "tbz x11, #1, 5f\n"
+      "ldr d11, [x9], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v11.s }[2], [x9]\n"
       "b 10f\n"
       "5:"  // Height 1: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s11, [x28, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s11, [x9, #0x0]\n"
       "b 10f\n"
       "6:"  // Height 1: Partial accumulate: partial_4_0
-      "tbz x10, #2, 8f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 7f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v10.s }[2], [x28]\n"
+      "tbz x11, #2, 8f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 7f\n"
+      "ldr d10, [x9], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v10.s }[2], [x9]\n"
       "b 10f\n"
       "7:"  // Height 1: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 10f\n"
-      "ldr s10, [x28, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 10f\n"
+      "ldr s10, [x9, #0x0]\n"
       "b 10f\n"
       "8:"  // Height 1: Partial accumulate: partial_2_0
-      "tbz x10, #1, 9f\n"
-      "ldr d9, [x28], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 10f\n"
-      "ld1 { v9.s }[2], [x28]\n"
+      "tbz x11, #1, 9f\n"
+      "ldr d9, [x9], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 10f\n"
+      "ld1 { v9.s }[2], [x9]\n"
       "b 10f\n"
       "9:"  // Height 1: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
+      "ldr s9, [x9, #0x0]\n"
+      "mov x25, #0x0\n"
       "10:"  // Height 1: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 12f\n"
       "11:"  // Height 1: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
       "12:"  // Height 1: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -174,337 +174,333 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "14:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 17f\n"
       "16:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "17:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 20f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 19f\n"
       "18:"  // Height 1: Multiply loop: Main loop head
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       "bge 18b\n"
       "19:"  // Height 1: Multiply loop: Single iteration only
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "20:"  // Height 1: Multiply loop: Main loop skip
-      "cbz x26, 27f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 27f\n"
+      "cmp x27, #0x8\n"
       "blt 22f\n"
       "21:"  // Height 1: Multiply loop: Odd block loop
-      "movi v2.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40f  // ummla v15.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       "bge 21b\n"
-      "cbz x26, 27f\n"
       "22:"  // Height 1: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 24f\n"
-      "ldr s1, [x25], #0x4\n"
-      "tbz x26, #1, 23f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[6], [x25]\n"
+      "cbz x27, 27f\n"
+      "tbz x27, #2, 24f\n"
+      "ldr s1, [x26], #0x4\n"
+      "tbz x27, #1, 23f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[6], [x26]\n"
       "b 26f\n"
       "23:"  // Height 1: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[4], [x25]\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[4], [x26]\n"
       "b 26f\n"
       "24:"  // Height 1: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 25f\n"
-      "ldr h1, [x25], #0x2\n"
-      "tbz x26, #0, 26f\n"
-      "ld1 { v1.b }[2], [x25]\n"
+      "tbz x27, #1, 25f\n"
+      "ldr h1, [x26], #0x2\n"
+      "tbz x27, #0, 26f\n"
+      "ld1 { v1.b }[2], [x26]\n"
       "b 26f\n"
       "25:"  // Height 1: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
       "26:"  // Height 1: Multiply loop: Ragged operand read: Done
-      "movi v2.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       "27:"  // Height 1: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 15b\n"
+      "cmp x11, #0x10\n"
       "uzp1 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
       "uzp1 v9.2d, v9.2d, v13.2d\n"
-      "cmp x10, #0x10\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp1 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v11.2d, v11.2d, v15.2d\n"
       "bge 36f\n"
-      "tbz x10, #3, 31f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "st1 { v9.4s }, [x28], #0x10\n"
-      "tbz x10, #2, 29f\n"
-      "st1 { v10.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 28f\n"
-      "str d11, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v11.s }[2], [x28]\n"
+      "tbz x11, #3, 31f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "st1 { v9.4s }, [x9], #0x10\n"
+      "tbz x11, #2, 29f\n"
+      "st1 { v10.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 28f\n"
+      "str d11, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v11.s }[2], [x9]\n"
       "b 35f\n"
       "28:"  // Height 1: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 35f\n"
-      "str s11, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s11, [x9, #0x0]\n"
       "b 35f\n"
       "29:"  // Height 1: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 30f\n"
-      "str d10, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v10.s }[2], [x28]\n"
+      "tbz x11, #1, 30f\n"
+      "str d10, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v10.s }[2], [x9]\n"
       "b 35f\n"
       "30:"  // Height 1: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 35f\n"
-      "str s10, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s10, [x9, #0x0]\n"
       "b 35f\n"
       "31:"  // Height 1: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 33f\n"
-      "st1 { v8.4s }, [x28], #0x10\n"
-      "tbz x10, #1, 32f\n"
-      "str d9, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v9.s }[2], [x28]\n"
+      "tbz x11, #2, 33f\n"
+      "st1 { v8.4s }, [x9], #0x10\n"
+      "tbz x11, #1, 32f\n"
+      "str d9, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v9.s }[2], [x9]\n"
       "b 35f\n"
       "32:"  // Height 1: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 35f\n"
-      "str s9, [x28, #0x0]\n"
+      "tbz x11, #0, 35f\n"
+      "str s9, [x9, #0x0]\n"
       "b 35f\n"
       "33:"  // Height 1: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 34f\n"
-      "str d8, [x28], #0x8\n"
-      "tbz x10, #0, 35f\n"
-      "st1 { v8.s }[2], [x28]\n"
+      "tbz x11, #1, 34f\n"
+      "str d8, [x9], #0x8\n"
+      "tbz x11, #0, 35f\n"
+      "st1 { v8.s }[2], [x9]\n"
       "b 35f\n"
       "34:"  // Height 1: Partial direct writeback: partial_1_0
-      "str s8, [x28, #0x0]\n"
+      "str s8, [x9, #0x0]\n"
       "35:"  // Height 1: Partial direct writeback: Done
       "b 37f\n"
       "36:"  // Height 1: Full writeback
-      "str q8, [x28, #0x0]\n"
-      "str q9, [x28, #0x10]\n"
-      "str q10, [x28, #0x20]\n"
-      "str q11, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
+      "str q8, [x9, #0x0]\n"
+      "str q9, [x9, #0x10]\n"
+      "str q10, [x9, #0x20]\n"
+      "str q11, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
       "37:"  // Height 1: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 2b\n"
       "b 224f\n"
       "38:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "39:"  // Height 2: Column loop
       "tbz %x[flags], #0, 50f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
+      "add x24, x9, x20, LSL #2\n"
       "bge 48f\n"
-      "tbz x10, #3, 43f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 41f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 40f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
+      "tbz x11, #3, 43f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 41f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 40f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
       "b 47f\n"
       "40:"  // Height 2: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
       "b 47f\n"
       "41:"  // Height 2: Partial accumulate: partial_2_8
-      "tbz x10, #1, 42f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
+      "tbz x11, #1, 42f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
       "b 47f\n"
       "42:"  // Height 2: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
       "b 47f\n"
       "43:"  // Height 2: Partial accumulate: partial_4_0
-      "tbz x10, #2, 45f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 44f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
+      "tbz x11, #2, 45f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 44f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
       "b 47f\n"
       "44:"  // Height 2: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 47f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 47f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
       "b 47f\n"
       "45:"  // Height 2: Partial accumulate: partial_2_0
-      "tbz x10, #1, 46f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "tbz x10, #0, 47f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
+      "tbz x11, #1, 46f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "tbz x11, #0, 47f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
       "b 47f\n"
       "46:"  // Height 2: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
       "47:"  // Height 2: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 49f\n"
       "48:"  // Height 2: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
       "49:"  // Height 2: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -525,398 +521,398 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
       "51:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "52:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 53f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 54f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 54f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 54f\n"
       "53:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "54:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 57f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 56f\n"
       "55:"  // Height 2: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "sub x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
+      "add x10, x10, #0x100\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       "bge 55b\n"
       "56:"  // Height 2: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "add x24, x24, #0x10\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
+      "trn2 v1.2d, v1.2d, v2.2d\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "add x10, x10, #0x100\n"
       "57:"  // Height 2: Multiply loop: Main loop skip
-      "cbz x26, 64f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 64f\n"
+      "cmp x27, #0x8\n"
       "blt 59f\n"
       "58:"  // Height 2: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "cmp x26, #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
-      "ldr q7, [x9, #0x10]\n"
-      "ldr q6, [x9, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x20]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x40]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x60]\n"
+      "ldr q7, [x10, #0x70]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40f  // ummla v15.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       "bge 58b\n"
-      "cbz x26, 64f\n"
       "59:"  // Height 2: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 61f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "tbz x26, #1, 60f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
+      "cbz x27, 64f\n"
+      "tbz x27, #2, 61f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "tbz x27, #1, 60f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
       "b 63f\n"
       "60:"  // Height 2: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
       "b 63f\n"
       "61:"  // Height 2: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 62f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "tbz x26, #0, 63f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
+      "tbz x27, #1, 62f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "tbz x27, #0, 63f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
       "b 63f\n"
       "62:"  // Height 2: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
       "63:"  // Height 2: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       "64:"  // Height 2: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 52b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "bge 73f\n"
-      "tbz x10, #3, 68f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "tbz x10, #2, 66f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 65f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
+      "tbz x11, #3, 68f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "tbz x11, #2, 66f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 65f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
       "b 72f\n"
       "65:"  // Height 2: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 72f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
       "b 72f\n"
       "66:"  // Height 2: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 67f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
+      "tbz x11, #1, 67f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
       "b 72f\n"
       "67:"  // Height 2: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 72f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
       "b 72f\n"
       "68:"  // Height 2: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 70f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "tbz x10, #1, 69f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
+      "tbz x11, #2, 70f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "tbz x11, #1, 69f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
       "b 72f\n"
       "69:"  // Height 2: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 72f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
+      "tbz x11, #0, 72f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
       "b 72f\n"
       "70:"  // Height 2: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 71f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "tbz x10, #0, 72f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
+      "tbz x11, #1, 71f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "tbz x11, #0, 72f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
       "b 72f\n"
       "71:"  // Height 2: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
       "72:"  // Height 2: Partial direct writeback: Done
       "b 74f\n"
       "73:"  // Height 2: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
       "74:"  // Height 2: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 39b\n"
       "b 224f\n"
       "75:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "76:"  // Height 3: Column loop
       "tbz %x[flags], #0, 87f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x23, x24, x20, LSL #2\n"
       "bge 85f\n"
-      "tbz x10, #3, 80f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 78f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 77f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
+      "tbz x11, #3, 80f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 78f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 77f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
       "b 84f\n"
       "77:"  // Height 3: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
       "b 84f\n"
       "78:"  // Height 3: Partial accumulate: partial_2_8
-      "tbz x10, #1, 79f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
+      "tbz x11, #1, 79f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
       "b 84f\n"
       "79:"  // Height 3: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
       "b 84f\n"
       "80:"  // Height 3: Partial accumulate: partial_4_0
-      "tbz x10, #2, 82f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 81f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
+      "tbz x11, #2, 82f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 81f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
       "b 84f\n"
       "81:"  // Height 3: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 84f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 84f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
       "b 84f\n"
       "82:"  // Height 3: Partial accumulate: partial_2_0
-      "tbz x10, #1, 83f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "tbz x10, #0, 84f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
+      "tbz x11, #1, 83f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "tbz x11, #0, 84f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
       "b 84f\n"
       "83:"  // Height 3: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
       "84:"  // Height 3: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 86f\n"
       "85:"  // Height 3: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
       "86:"  // Height 3: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -953,285 +949,281 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "88:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "89:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 90f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 91f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 91f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 91f\n"
       "90:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "91:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 94f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 93f\n"
       "92:"  // Height 3: Multiply loop: Main loop head
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "cmp x26, #0x20\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 92b\n"
       "93:"  // Height 3: Multiply loop: Single iteration only
-      "movi v4.16b, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
       "94:"  // Height 3: Multiply loop: Main loop skip
-      "cbz x26, 101f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 101f\n"
+      "cmp x27, #0x8\n"
       "blt 96f\n"
       "95:"  // Height 3: Multiply loop: Odd block loop
-      "movi v4.16b, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a450  // ummla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a454  // ummla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6e86a451  // ummla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e87a455  // ummla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a452  // ummla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a456  // ummla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e86a453  // ummla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a40f  // ummla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       "bge 95b\n"
-      "cbz x26, 101f\n"
       "96:"  // Height 3: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 98f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "tbz x26, #1, 97f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
+      "cbz x27, 101f\n"
+      "tbz x27, #2, 98f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "tbz x27, #1, 97f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
       "b 100f\n"
       "97:"  // Height 3: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
       "b 100f\n"
       "98:"  // Height 3: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 99f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "tbz x26, #0, 100f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
+      "tbz x27, #1, 99f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "tbz x27, #0, 100f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
       "b 100f\n"
       "99:"  // Height 3: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
       "100:"  // Height 3: Multiply loop: Ragged operand read: Done
-      "movi v4.16b, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       "101:"  // Height 3: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 89b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v16.2d, v16.2d, v20.2d\n"
@@ -1239,234 +1231,234 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "uzp1 v18.2d, v18.2d, v22.2d\n"
       "uzp1 v19.2d, v19.2d, v23.2d\n"
       "bge 110f\n"
-      "tbz x10, #3, 105f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "st1 { v17.4s }, [x22], #0x10\n"
-      "tbz x10, #2, 103f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v18.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 102f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d19, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v19.s }[2], [x22]\n"
+      "tbz x11, #3, 105f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "st1 { v17.4s }, [x23], #0x10\n"
+      "tbz x11, #2, 103f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v18.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 102f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d19, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v19.s }[2], [x23]\n"
       "b 109f\n"
       "102:"  // Height 3: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 109f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s19, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s19, [x23, #0x0]\n"
       "b 109f\n"
       "103:"  // Height 3: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 104f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d18, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v18.s }[2], [x22]\n"
+      "tbz x11, #1, 104f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d18, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v18.s }[2], [x23]\n"
       "b 109f\n"
       "104:"  // Height 3: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 109f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s18, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s18, [x23, #0x0]\n"
       "b 109f\n"
       "105:"  // Height 3: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 107f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v16.4s }, [x22], #0x10\n"
-      "tbz x10, #1, 106f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d17, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v17.s }[2], [x22]\n"
+      "tbz x11, #2, 107f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v16.4s }, [x23], #0x10\n"
+      "tbz x11, #1, 106f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d17, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v17.s }[2], [x23]\n"
       "b 109f\n"
       "106:"  // Height 3: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 109f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s17, [x22, #0x0]\n"
+      "tbz x11, #0, 109f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s17, [x23, #0x0]\n"
       "b 109f\n"
       "107:"  // Height 3: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 108f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d16, [x22], #0x8\n"
-      "tbz x10, #0, 109f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v16.s }[2], [x22]\n"
+      "tbz x11, #1, 108f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d16, [x23], #0x8\n"
+      "tbz x11, #0, 109f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v16.s }[2], [x23]\n"
       "b 109f\n"
       "108:"  // Height 3: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s16, [x22, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s16, [x23, #0x0]\n"
       "109:"  // Height 3: Partial direct writeback: Done
       "b 111f\n"
       "110:"  // Height 3: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q16, [x22, #0x0]\n"
-      "str q17, [x22, #0x10]\n"
-      "str q18, [x22, #0x20]\n"
-      "str q19, [x22, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q16, [x23, #0x0]\n"
+      "str q17, [x23, #0x10]\n"
+      "str q18, [x23, #0x20]\n"
+      "str q19, [x23, #0x30]\n"
       "111:"  // Height 3: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 76b\n"
       "b 224f\n"
       "112:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "113:"  // Height 4: Column loop
       "tbz %x[flags], #0, 124f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x22, x23, x20, LSL #2\n"
       "bge 122f\n"
-      "tbz x10, #3, 117f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 115f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 114f\n"
-      "mov x24, #0x38\n"
-      "ldr d16, [x28], #0x8\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
+      "tbz x11, #3, 117f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 115f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 114f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
       "b 121f\n"
       "114:"  // Height 4: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
       "b 121f\n"
       "115:"  // Height 4: Partial accumulate: partial_2_8
-      "tbz x10, #1, 116f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
+      "tbz x11, #1, 116f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
       "b 121f\n"
       "116:"  // Height 4: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
       "b 121f\n"
       "117:"  // Height 4: Partial accumulate: partial_4_0
-      "tbz x10, #2, 119f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 118f\n"
-      "mov x24, #0x18\n"
-      "ldr d10, [x28], #0x8\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
+      "tbz x11, #2, 119f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 118f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
       "b 121f\n"
       "118:"  // Height 4: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 121f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 121f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
       "b 121f\n"
       "119:"  // Height 4: Partial accumulate: partial_2_0
-      "tbz x10, #1, 120f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "tbz x10, #0, 121f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
+      "tbz x11, #1, 120f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "tbz x11, #0, 121f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
       "b 121f\n"
       "120:"  // Height 4: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
       "121:"  // Height 4: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 123f\n"
       "122:"  // Height 4: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
       "123:"  // Height 4: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -1503,301 +1495,301 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "125:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "126:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 127f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 128f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 128f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 128f\n"
       "127:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "128:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 131f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "blt 130f\n"
       "129:"  // Height 4: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x23, x23, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
+      "sub x27, x27, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "cmp x26, #0x20\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "bge 129b\n"
       "130:"  // Height 4: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
-      "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x24, x24, #0x10\n"
-      "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x23, x23, #0x10\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x22, x22, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "trn1 v2.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "trn2 v3.2d, v3.2d, v4.2d\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
       "131:"  // Height 4: Multiply loop: Main loop skip
-      "cbz x26, 138f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 138f\n"
+      "cmp x27, #0x8\n"
       "blt 133f\n"
       "132:"  // Height 4: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
+      "cmp x27, #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a450  // ummla v16.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a454  // ummla v20.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a451  // ummla v17.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a455  // ummla v21.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a452  // ummla v18.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a456  // ummla v22.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a453  // ummla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a40f  // ummla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       "bge 132b\n"
-      "cbz x26, 138f\n"
       "133:"  // Height 4: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 135f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "tbz x26, #1, 134f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
+      "cbz x27, 138f\n"
+      "tbz x27, #2, 135f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "tbz x27, #1, 134f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
       "b 137f\n"
       "134:"  // Height 4: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
       "b 137f\n"
       "135:"  // Height 4: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 136f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "tbz x26, #0, 137f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
+      "tbz x27, #1, 136f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "tbz x27, #0, 137f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
       "b 137f\n"
       "136:"  // Height 4: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
       "137:"  // Height 4: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
+      "ldr q6, [x10, #0x10]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       "138:"  // Height 4: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 126b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
@@ -1808,275 +1800,275 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "uzp1 v22.2d, v19.2d, v23.2d\n"
       "uzp2 v19.2d, v19.2d, v23.2d\n"
       "bge 147f\n"
-      "tbz x10, #3, 142f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "tbz x10, #2, 140f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 139f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
+      "tbz x11, #3, 142f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "tbz x11, #2, 140f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 139f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
       "b 146f\n"
       "139:"  // Height 4: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 146f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
       "b 146f\n"
       "140:"  // Height 4: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 141f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
+      "tbz x11, #1, 141f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
       "b 146f\n"
       "141:"  // Height 4: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 146f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
       "b 146f\n"
       "142:"  // Height 4: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 144f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "tbz x10, #1, 143f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
+      "tbz x11, #2, 144f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "tbz x11, #1, 143f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
       "b 146f\n"
       "143:"  // Height 4: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 146f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
+      "tbz x11, #0, 146f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
       "b 146f\n"
       "144:"  // Height 4: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 145f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "tbz x10, #0, 146f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
+      "tbz x11, #1, 145f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "tbz x11, #0, 146f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
       "b 146f\n"
       "145:"  // Height 4: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
       "146:"  // Height 4: Partial direct writeback: Done
       "b 148f\n"
       "147:"  // Height 4: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
       "148:"  // Height 4: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 113b\n"
       "b 224f\n"
       "149:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "150:"  // Height 5: Column loop
       "tbz %x[flags], #0, 161f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x21, x22, x20, LSL #2\n"
       "bge 159f\n"
-      "tbz x10, #3, 154f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 152f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 151f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d6, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v6.s }[2], [x20]\n"
+      "tbz x11, #3, 154f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 152f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 151f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d6, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v6.s }[2], [x21]\n"
       "b 158f\n"
       "151:"  // Height 5: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s6, [x20, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s6, [x21, #0x0]\n"
       "b 158f\n"
       "152:"  // Height 5: Partial accumulate: partial_2_8
-      "tbz x10, #1, 153f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
+      "tbz x11, #1, 153f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
       "b 158f\n"
       "153:"  // Height 5: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
       "b 158f\n"
       "154:"  // Height 5: Partial accumulate: partial_4_0
-      "tbz x10, #2, 156f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 155f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
+      "tbz x11, #2, 156f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 155f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
       "b 158f\n"
       "155:"  // Height 5: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 158f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 158f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
       "b 158f\n"
       "156:"  // Height 5: Partial accumulate: partial_2_0
-      "tbz x10, #1, 157f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "tbz x10, #0, 158f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
+      "tbz x11, #1, 157f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "tbz x11, #0, 158f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
       "b 158f\n"
       "157:"  // Height 5: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
       "158:"  // Height 5: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 160f\n"
       "159:"  // Height 5: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q25, [x20, #0x0]\n"
-      "ldr q26, [x20, #0x10]\n"
-      "ldr q27, [x20, #0x20]\n"
-      "ldr q6, [x20, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q25, [x21, #0x0]\n"
+      "ldr q26, [x21, #0x10]\n"
+      "ldr q27, [x21, #0x20]\n"
+      "ldr q6, [x21, #0x30]\n"
       "160:"  // Height 5: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2129,212 +2121,210 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "162:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "163:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 164f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 165f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 165f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 165f\n"
       "164:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "165:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 168f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 167f\n"
       "166:"  // Height 5: Multiply loop: Main loop head
-      "movi v6.4s, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "cmp x26, #0x20\n"
-      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
+      "ldr q2, [x25, #0x0]\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b8  // ummla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bc  // ummla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b9  // ummla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bd  // ummla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4ba  // ummla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4be  // ummla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4bb  // ummla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e86a4bf  // ummla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
       "bge 166b\n"
       "167:"  // Height 5: Multiply loop: Single iteration only
-      "movi v6.4s, #0x0\n"
-      "ldr q2, [x24, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b8  // ummla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bc  // ummla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b9  // ummla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bd  // ummla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4ba  // ummla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4be  // ummla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4bb  // ummla v27.4s, v5.16b, v7.16b\n"
@@ -2342,136 +2332,134 @@ void a64_hybrid_u8u32_mmla_6x16 (
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bf  // ummla v31.4s, v5.16b, v6.16b\n"
       "168:"  // Height 5: Multiply loop: Main loop skip
-      "cbz x26, 175f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 175f\n"
+      "cmp x27, #0x8\n"
       "blt 170f\n"
       "169:"  // Height 5: Multiply loop: Odd block loop
-      "movi v7.4s, #0x0\n"
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr q6, [x9, #0x0]\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr q6, [x10, #0x0]\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q7, [x9, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a450  // ummla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a498  // ummla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a454  // ummla v20.4s, v2.16b, v7.16b\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6e87a49c  // ummla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a451  // ummla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a499  // ummla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a455  // ummla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49d  // ummla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a452  // ummla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49a  // ummla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a456  // ummla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49e  // ummla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e86a453  // ummla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49b  // ummla v27.4s, v4.16b, v6.16b\n"
       ".inst 0x6e87a40f  // ummla v15.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49f  // ummla v31.4s, v4.16b, v7.16b\n"
       "bge 169b\n"
-      "cbz x26, 175f\n"
       "170:"  // Height 5: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 172f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "tbz x26, #1, 171f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
+      "cbz x27, 175f\n"
+      "tbz x27, #2, 172f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "tbz x27, #1, 171f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
       "b 174f\n"
       "171:"  // Height 5: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
       "b 174f\n"
       "172:"  // Height 5: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 173f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "tbz x26, #0, 174f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
+      "tbz x27, #1, 173f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "tbz x27, #0, 174f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
       "b 174f\n"
       "173:"  // Height 5: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
       "174:"  // Height 5: Multiply loop: Ragged operand read: Done
-      "movi v6.4s, #0x0\n"
-      "ldr q7, [x9, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
@@ -2479,30 +2467,30 @@ void a64_hybrid_u8u32_mmla_6x16 (
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
       "175:"  // Height 5: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 163b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
@@ -2515,319 +2503,319 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "uzp1 v26.2d, v26.2d, v30.2d\n"
       "uzp1 v27.2d, v27.2d, v31.2d\n"
       "bge 184f\n"
-      "tbz x10, #3, 179f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "st1 { v25.4s }, [x20], #0x10\n"
-      "tbz x10, #2, 177f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "st1 { v26.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 176f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d27, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v27.s }[2], [x20]\n"
+      "tbz x11, #3, 179f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "st1 { v25.4s }, [x21], #0x10\n"
+      "tbz x11, #2, 177f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v26.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 176f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d27, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v27.s }[2], [x21]\n"
       "b 183f\n"
       "176:"  // Height 5: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 183f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s27, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s27, [x21, #0x0]\n"
       "b 183f\n"
       "177:"  // Height 5: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 178f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "str d26, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
-      "st1 { v26.s }[2], [x20]\n"
+      "tbz x11, #1, 178f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "str d26, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v26.s }[2], [x21]\n"
       "b 183f\n"
       "178:"  // Height 5: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 183f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
-      "str s26, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
+      "str s26, [x21, #0x0]\n"
       "b 183f\n"
       "179:"  // Height 5: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 181f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v24.4s }, [x20], #0x10\n"
-      "tbz x10, #1, 180f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "str d25, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
-      "st1 { v25.s }[2], [x20]\n"
+      "tbz x11, #2, 181f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v24.4s }, [x21], #0x10\n"
+      "tbz x11, #1, 180f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "str d25, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v25.s }[2], [x21]\n"
       "b 183f\n"
       "180:"  // Height 5: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 183f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
-      "str s25, [x20, #0x0]\n"
+      "tbz x11, #0, 183f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
+      "str s25, [x21, #0x0]\n"
       "b 183f\n"
       "181:"  // Height 5: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 182f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d24, [x20], #0x8\n"
-      "tbz x10, #0, 183f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
-      "st1 { v24.s }[2], [x20]\n"
+      "tbz x11, #1, 182f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d24, [x21], #0x8\n"
+      "tbz x11, #0, 183f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v24.s }[2], [x21]\n"
       "b 183f\n"
       "182:"  // Height 5: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
-      "str s24, [x20, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
+      "str s24, [x21, #0x0]\n"
       "183:"  // Height 5: Partial direct writeback: Done
       "b 185f\n"
       "184:"  // Height 5: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
-      "str q24, [x20, #0x0]\n"
-      "str q25, [x20, #0x10]\n"
-      "str q26, [x20, #0x20]\n"
-      "str q27, [x20, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
+      "str q24, [x21, #0x0]\n"
+      "str q25, [x21, #0x10]\n"
+      "str q26, [x21, #0x20]\n"
+      "str q27, [x21, #0x30]\n"
       "185:"  // Height 5: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 150b\n"
       "b 224f\n"
       "186:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "187:"  // Height 6: Column loop
       "tbz %x[flags], #0, 198f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "cmp x10, #0x10\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
+      "add x20, x21, x20, LSL #2\n"
       "bge 196f\n"
-      "tbz x10, #3, 191f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v10.4s }, [x28], #0x10\n"
-      "ld1 { v13.4s }, [x23], #0x10\n"
-      "ld1 { v18.4s }, [x22], #0x10\n"
-      "ld1 { v21.4s }, [x21], #0x10\n"
-      "ld1 { v26.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "ld1 { v29.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 189f\n"
-      "ld1 { v11.4s }, [x28], #0x10\n"
-      "ld1 { v14.4s }, [x23], #0x10\n"
-      "ld1 { v19.4s }, [x22], #0x10\n"
-      "ld1 { v22.4s }, [x21], #0x10\n"
-      "ld1 { v27.4s }, [x20], #0x10\n"
-      "ld1 { v30.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 188f\n"
-      "ldr d16, [x28], #0x8\n"
-      "mov x24, #0x38\n"
-      "ldr d15, [x23], #0x8\n"
-      "ldr d24, [x22], #0x8\n"
-      "ldr d23, [x21], #0x8\n"
-      "ldr d6, [x20], #0x8\n"
-      "ldr d31, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v16.s }[2], [x28]\n"
-      "ld1 { v15.s }[2], [x23]\n"
-      "ld1 { v24.s }[2], [x22]\n"
-      "ld1 { v23.s }[2], [x21]\n"
-      "ld1 { v6.s }[2], [x20]\n"
-      "ld1 { v31.s }[2], [x19]\n"
+      "tbz x11, #3, 191f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "ld1 { v10.4s }, [x9], #0x10\n"
+      "ld1 { v13.4s }, [x24], #0x10\n"
+      "ld1 { v18.4s }, [x23], #0x10\n"
+      "ld1 { v21.4s }, [x22], #0x10\n"
+      "ld1 { v26.4s }, [x21], #0x10\n"
+      "ld1 { v29.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 189f\n"
+      "ld1 { v11.4s }, [x9], #0x10\n"
+      "ld1 { v14.4s }, [x24], #0x10\n"
+      "ld1 { v19.4s }, [x23], #0x10\n"
+      "ld1 { v22.4s }, [x22], #0x10\n"
+      "ld1 { v27.4s }, [x21], #0x10\n"
+      "ld1 { v30.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 188f\n"
+      "ldr d16, [x9], #0x8\n"
+      "ldr d15, [x24], #0x8\n"
+      "mov x25, #0x38\n"
+      "ldr d24, [x23], #0x8\n"
+      "ldr d23, [x22], #0x8\n"
+      "ldr d6, [x21], #0x8\n"
+      "ldr d31, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v16.s }[2], [x9]\n"
+      "ld1 { v15.s }[2], [x24]\n"
+      "ld1 { v24.s }[2], [x23]\n"
+      "ld1 { v23.s }[2], [x22]\n"
+      "ld1 { v6.s }[2], [x21]\n"
+      "ld1 { v31.s }[2], [x20]\n"
       "b 195f\n"
       "188:"  // Height 6: Partial accumulate: partial_1_12
-      "mov x24, #0x30\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s16, [x28, #0x0]\n"
-      "ldr s15, [x23, #0x0]\n"
-      "ldr s24, [x22, #0x0]\n"
-      "ldr s23, [x21, #0x0]\n"
-      "ldr s6, [x20, #0x0]\n"
-      "ldr s31, [x19, #0x0]\n"
+      "mov x25, #0x30\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s16, [x9, #0x0]\n"
+      "ldr s15, [x24, #0x0]\n"
+      "ldr s24, [x23, #0x0]\n"
+      "ldr s23, [x22, #0x0]\n"
+      "ldr s6, [x21, #0x0]\n"
+      "ldr s31, [x20, #0x0]\n"
       "b 195f\n"
       "189:"  // Height 6: Partial accumulate: partial_2_8
-      "tbz x10, #1, 190f\n"
-      "ldr d11, [x28], #0x8\n"
-      "ldr d14, [x23], #0x8\n"
-      "mov x24, #0x28\n"
-      "ldr d19, [x22], #0x8\n"
-      "ldr d22, [x21], #0x8\n"
-      "ldr d27, [x20], #0x8\n"
-      "ldr d30, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v11.s }[2], [x28]\n"
-      "ld1 { v14.s }[2], [x23]\n"
-      "ld1 { v19.s }[2], [x22]\n"
-      "ld1 { v22.s }[2], [x21]\n"
-      "ld1 { v27.s }[2], [x20]\n"
-      "ld1 { v30.s }[2], [x19]\n"
+      "tbz x11, #1, 190f\n"
+      "ldr d11, [x9], #0x8\n"
+      "ldr d14, [x24], #0x8\n"
+      "mov x25, #0x28\n"
+      "ldr d19, [x23], #0x8\n"
+      "ldr d22, [x22], #0x8\n"
+      "ldr d27, [x21], #0x8\n"
+      "ldr d30, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v11.s }[2], [x9]\n"
+      "ld1 { v14.s }[2], [x24]\n"
+      "ld1 { v19.s }[2], [x23]\n"
+      "ld1 { v22.s }[2], [x22]\n"
+      "ld1 { v27.s }[2], [x21]\n"
+      "ld1 { v30.s }[2], [x20]\n"
       "b 195f\n"
       "190:"  // Height 6: Partial accumulate: partial_1_8
-      "mov x24, #0x20\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s11, [x28, #0x0]\n"
-      "ldr s14, [x23, #0x0]\n"
-      "ldr s19, [x22, #0x0]\n"
-      "ldr s22, [x21, #0x0]\n"
-      "ldr s27, [x20, #0x0]\n"
-      "ldr s30, [x19, #0x0]\n"
+      "mov x25, #0x20\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s11, [x9, #0x0]\n"
+      "ldr s14, [x24, #0x0]\n"
+      "ldr s19, [x23, #0x0]\n"
+      "ldr s22, [x22, #0x0]\n"
+      "ldr s27, [x21, #0x0]\n"
+      "ldr s30, [x20, #0x0]\n"
       "b 195f\n"
       "191:"  // Height 6: Partial accumulate: partial_4_0
-      "tbz x10, #2, 193f\n"
-      "ld1 { v9.4s }, [x28], #0x10\n"
-      "ld1 { v12.4s }, [x23], #0x10\n"
-      "ld1 { v17.4s }, [x22], #0x10\n"
-      "ld1 { v20.4s }, [x21], #0x10\n"
-      "ld1 { v25.4s }, [x20], #0x10\n"
-      "ld1 { v28.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 192f\n"
-      "ldr d10, [x28], #0x8\n"
-      "mov x24, #0x18\n"
-      "ldr d13, [x23], #0x8\n"
-      "ldr d18, [x22], #0x8\n"
-      "ldr d21, [x21], #0x8\n"
-      "ldr d26, [x20], #0x8\n"
-      "ldr d29, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v10.s }[2], [x28]\n"
-      "ld1 { v13.s }[2], [x23]\n"
-      "ld1 { v18.s }[2], [x22]\n"
-      "ld1 { v21.s }[2], [x21]\n"
-      "ld1 { v26.s }[2], [x20]\n"
-      "ld1 { v29.s }[2], [x19]\n"
+      "tbz x11, #2, 193f\n"
+      "ld1 { v9.4s }, [x9], #0x10\n"
+      "ld1 { v12.4s }, [x24], #0x10\n"
+      "ld1 { v17.4s }, [x23], #0x10\n"
+      "ld1 { v20.4s }, [x22], #0x10\n"
+      "ld1 { v25.4s }, [x21], #0x10\n"
+      "ld1 { v28.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 192f\n"
+      "ldr d10, [x9], #0x8\n"
+      "ldr d13, [x24], #0x8\n"
+      "mov x25, #0x18\n"
+      "ldr d18, [x23], #0x8\n"
+      "ldr d21, [x22], #0x8\n"
+      "ldr d26, [x21], #0x8\n"
+      "ldr d29, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v10.s }[2], [x9]\n"
+      "ld1 { v13.s }[2], [x24]\n"
+      "ld1 { v18.s }[2], [x23]\n"
+      "ld1 { v21.s }[2], [x22]\n"
+      "ld1 { v26.s }[2], [x21]\n"
+      "ld1 { v29.s }[2], [x20]\n"
       "b 195f\n"
       "192:"  // Height 6: Partial accumulate: partial_1_4
-      "mov x24, #0x10\n"
-      "tbz x10, #0, 195f\n"
-      "ldr s10, [x28, #0x0]\n"
-      "ldr s13, [x23, #0x0]\n"
-      "ldr s18, [x22, #0x0]\n"
-      "ldr s21, [x21, #0x0]\n"
-      "ldr s26, [x20, #0x0]\n"
-      "ldr s29, [x19, #0x0]\n"
+      "mov x25, #0x10\n"
+      "tbz x11, #0, 195f\n"
+      "ldr s10, [x9, #0x0]\n"
+      "ldr s13, [x24, #0x0]\n"
+      "ldr s18, [x23, #0x0]\n"
+      "ldr s21, [x22, #0x0]\n"
+      "ldr s26, [x21, #0x0]\n"
+      "ldr s29, [x20, #0x0]\n"
       "b 195f\n"
       "193:"  // Height 6: Partial accumulate: partial_2_0
-      "tbz x10, #1, 194f\n"
-      "ldr d9, [x28], #0x8\n"
-      "ldr d12, [x23], #0x8\n"
-      "mov x24, #0x8\n"
-      "ldr d17, [x22], #0x8\n"
-      "ldr d20, [x21], #0x8\n"
-      "ldr d25, [x20], #0x8\n"
-      "ldr d28, [x19], #0x8\n"
-      "tbz x10, #0, 195f\n"
-      "ld1 { v9.s }[2], [x28]\n"
-      "ld1 { v12.s }[2], [x23]\n"
-      "ld1 { v17.s }[2], [x22]\n"
-      "ld1 { v20.s }[2], [x21]\n"
-      "ld1 { v25.s }[2], [x20]\n"
-      "ld1 { v28.s }[2], [x19]\n"
+      "tbz x11, #1, 194f\n"
+      "ldr d9, [x9], #0x8\n"
+      "ldr d12, [x24], #0x8\n"
+      "mov x25, #0x8\n"
+      "ldr d17, [x23], #0x8\n"
+      "ldr d20, [x22], #0x8\n"
+      "ldr d25, [x21], #0x8\n"
+      "ldr d28, [x20], #0x8\n"
+      "tbz x11, #0, 195f\n"
+      "ld1 { v9.s }[2], [x9]\n"
+      "ld1 { v12.s }[2], [x24]\n"
+      "ld1 { v17.s }[2], [x23]\n"
+      "ld1 { v20.s }[2], [x22]\n"
+      "ld1 { v25.s }[2], [x21]\n"
+      "ld1 { v28.s }[2], [x20]\n"
       "b 195f\n"
       "194:"  // Height 6: Partial accumulate: partial_1_0
-      "ldr s9, [x28, #0x0]\n"
-      "mov x24, #0x0\n"
-      "ldr s12, [x23, #0x0]\n"
-      "ldr s17, [x22, #0x0]\n"
-      "ldr s20, [x21, #0x0]\n"
-      "ldr s25, [x20, #0x0]\n"
-      "ldr s28, [x19, #0x0]\n"
+      "ldr s9, [x9, #0x0]\n"
+      "ldr s12, [x24, #0x0]\n"
+      "mov x25, #0x0\n"
+      "ldr s17, [x23, #0x0]\n"
+      "ldr s20, [x22, #0x0]\n"
+      "ldr s25, [x21, #0x0]\n"
+      "ldr s28, [x20, #0x0]\n"
       "195:"  // Height 6: Partial accumulate: Done
-      "sub x28, x28, x24\n"
+      "sub x9, x9, x25\n"
       "b 197f\n"
       "196:"  // Height 6: full accumulate
-      "ldr q9, [x28, #0x0]\n"
-      "ldr q10, [x28, #0x10]\n"
-      "ldr q11, [x28, #0x20]\n"
-      "ldr q16, [x28, #0x30]\n"
-      "ldr q12, [x23, #0x0]\n"
-      "ldr q13, [x23, #0x10]\n"
-      "ldr q14, [x23, #0x20]\n"
-      "ldr q15, [x23, #0x30]\n"
-      "ldr q17, [x22, #0x0]\n"
-      "ldr q18, [x22, #0x10]\n"
-      "ldr q19, [x22, #0x20]\n"
-      "ldr q24, [x22, #0x30]\n"
-      "ldr q20, [x21, #0x0]\n"
-      "ldr q21, [x21, #0x10]\n"
-      "ldr q22, [x21, #0x20]\n"
-      "ldr q23, [x21, #0x30]\n"
-      "ldr q25, [x20, #0x0]\n"
-      "ldr q26, [x20, #0x10]\n"
-      "ldr q27, [x20, #0x20]\n"
-      "ldr q6, [x20, #0x30]\n"
-      "ldr q28, [x19, #0x0]\n"
-      "ldr q29, [x19, #0x10]\n"
-      "ldr q30, [x19, #0x20]\n"
-      "ldr q31, [x19, #0x30]\n"
+      "ldr q9, [x9, #0x0]\n"
+      "ldr q10, [x9, #0x10]\n"
+      "ldr q11, [x9, #0x20]\n"
+      "ldr q16, [x9, #0x30]\n"
+      "ldr q12, [x24, #0x0]\n"
+      "ldr q13, [x24, #0x10]\n"
+      "ldr q14, [x24, #0x20]\n"
+      "ldr q15, [x24, #0x30]\n"
+      "ldr q17, [x23, #0x0]\n"
+      "ldr q18, [x23, #0x10]\n"
+      "ldr q19, [x23, #0x20]\n"
+      "ldr q24, [x23, #0x30]\n"
+      "ldr q20, [x22, #0x0]\n"
+      "ldr q21, [x22, #0x10]\n"
+      "ldr q22, [x22, #0x20]\n"
+      "ldr q23, [x22, #0x30]\n"
+      "ldr q25, [x21, #0x0]\n"
+      "ldr q26, [x21, #0x10]\n"
+      "ldr q27, [x21, #0x20]\n"
+      "ldr q6, [x21, #0x30]\n"
+      "ldr q28, [x20, #0x0]\n"
+      "ldr q29, [x20, #0x10]\n"
+      "ldr q30, [x20, #0x20]\n"
+      "ldr q31, [x20, #0x30]\n"
       "197:"  // Height 6: MMLA fixup
       "zip1 v8.2d, v9.2d, v12.2d\n"
       "zip2 v12.2d, v9.2d, v12.2d\n"
@@ -2880,219 +2868,219 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "movi v30.4s, #0x0\n"
       "movi v31.4s, #0x0\n"
       "199:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "200:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 201f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 202f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 202f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 202f\n"
       "201:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "202:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "blt 205f\n"
-      "ldr q1, [x25, #0x0]\n"
-      "ldr q2, [x24, #0x0]\n"
-      "cmp x26, #0x20\n"
+      "ldr q1, [x26, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "cmp x27, #0x20\n"
+      "ldr q3, [x24, #0x0]\n"
+      "ldr q4, [x23, #0x0]\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
+      "ldr q7, [x10, #0x0]\n"
       "blt 204f\n"
       "203:"  // Height 6: Multiply loop: Main loop head
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "add x25, x25, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x24, x24, #0x10\n"
+      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
+      "sub x27, x27, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x23, x23, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x21, x21, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "sub x26, x26, #0x10\n"
-      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
-      "cmp x26, #0x20\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      "add x22, x22, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "cmp x27, #0x20\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
-      "ldr q2, [x24, #0x0]\n"
+      "ldr q2, [x25, #0x0]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
+      "ldr q4, [x23, #0x0]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b8  // ummla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bc  // ummla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b9  // ummla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bd  // ummla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4ba  // ummla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4be  // ummla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4bb  // ummla v27.4s, v5.16b, v7.16b\n"
+      "ldr q7, [x10, #0x0]\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
-      "ldr q1, [x25, #0x0]\n"
+      "ldr q1, [x26, #0x0]\n"
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
+      "ldr q3, [x24, #0x0]\n"
       ".inst 0x6e86a4bf  // ummla v31.4s, v5.16b, v6.16b\n"
+      "ldr q5, [x22, #0x0]\n"
+      "ldr q6, [x21, #0x0]\n"
       "bge 203b\n"
       "204:"  // Height 6: Multiply loop: Single iteration only
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q3, [x23, #0x0]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 v1.2d, v1.2d, v2.2d\n"
-      "ldr q4, [x22, #0x0]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
+      "add x26, x26, #0x10\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr q5, [x21, #0x0]\n"
-      "add x24, x24, #0x10\n"
       "trn2 v3.2d, v3.2d, v4.2d\n"
-      "ldr q6, [x20, #0x0]\n"
-      "add x23, x23, #0x10\n"
+      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
+      "add x25, x25, #0x10\n"
       "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q7, [x9, #0x0]\n"
-      "add x22, x22, #0x10\n"
       "trn2 v5.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
-      "prfm pldl1keep, [x25, #0x80]\n"
-      "add x20, x20, #0x10\n"
-      ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x24, #0x80]\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
-      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "prfm pldl1keep, [x22, #0x80]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
-      "prfm pldl1keep, [x21, #0x80]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      "add x21, x21, #0x10\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x26, #0x80]\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
+      "prfm pldl1keep, [x25, #0x80]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
+      "prfm pldl1keep, [x24, #0x80]\n"
+      "prfm pldl1keep, [x23, #0x80]\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x80]\n"
+      "ldr q7, [x10, #0x80]\n"
+      "prfm pldl1keep, [x21, #0x80]\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x90]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6e87a428  // ummla v8.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a470  // ummla v16.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b8  // ummla v24.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xa0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6e86a42c  // ummla v12.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a474  // ummla v20.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bc  // ummla v28.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xb0]\n"
+      "ldr q6, [x10, #0xb0]\n"
       ".inst 0x6e87a429  // ummla v9.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a471  // ummla v17.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4b9  // ummla v25.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xc0]\n"
+      "ldr q7, [x10, #0xc0]\n"
       ".inst 0x6e86a42d  // ummla v13.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a475  // ummla v21.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bd  // ummla v29.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xd0]\n"
+      "ldr q6, [x10, #0xd0]\n"
       ".inst 0x6e87a42a  // ummla v10.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a472  // ummla v18.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4ba  // ummla v26.4s, v5.16b, v7.16b\n"
-      "ldr q7, [x9, #0xe0]\n"
+      "ldr q7, [x10, #0xe0]\n"
       ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e86a476  // ummla v22.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4be  // ummla v30.4s, v5.16b, v6.16b\n"
-      "ldr q6, [x9, #0xf0]\n"
-      "add x9, x9, #0x100\n"
+      "ldr q6, [x10, #0xf0]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6e87a42b  // ummla v11.4s, v1.16b, v7.16b\n"
       ".inst 0x6e87a473  // ummla v19.4s, v3.16b, v7.16b\n"
       ".inst 0x6e87a4bb  // ummla v27.4s, v5.16b, v7.16b\n"
@@ -3100,48 +3088,48 @@ void a64_hybrid_u8u32_mmla_6x16 (
       ".inst 0x6e86a477  // ummla v23.4s, v3.16b, v6.16b\n"
       ".inst 0x6e86a4bf  // ummla v31.4s, v5.16b, v6.16b\n"
       "205:"  // Height 6: Multiply loop: Main loop skip
-      "cbz x26, 212f\n"
-      "cmp x26, #0x8\n"
+      "cbz x27, 212f\n"
+      "cmp x27, #0x8\n"
       "blt 207f\n"
       "206:"  // Height 6: Multiply loop: Odd block loop
-      "ldr d1, [x25], #0x8\n"
-      "sub x26, x26, #0x8\n"
-      "ldr d2, [x24], #0x8\n"
+      "ldr d1, [x26], #0x8\n"
+      "ldr d2, [x25], #0x8\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr d3, [x23], #0x8\n"
-      "cmp x26, #0x8\n"
-      "ldr d4, [x22], #0x8\n"
+      "sub x27, x27, #0x8\n"
+      "ldr d3, [x24], #0x8\n"
+      "ldr d4, [x23], #0x8\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "ldr d5, [x21], #0x8\n"
-      "ldr d7, [x20], #0x8\n"
+      "cmp x27, #0x8\n"
+      "ldr d5, [x22], #0x8\n"
+      "ldr d7, [x21], #0x8\n"
       "trn1 v4.2d, v5.2d, v7.2d\n"
-      "ldr q6, [x9, #0x0]\n"
-      "ldr q7, [x9, #0x10]\n"
+      "ldr q6, [x10, #0x0]\n"
+      "ldr q7, [x10, #0x10]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a450  // ummla v16.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a498  // ummla v24.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x20]\n"
+      "ldr q6, [x10, #0x20]\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a454  // ummla v20.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49c  // ummla v28.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x30]\n"
+      "ldr q7, [x10, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a451  // ummla v17.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a499  // ummla v25.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x40]\n"
+      "ldr q6, [x10, #0x40]\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a455  // ummla v21.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49d  // ummla v29.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x50]\n"
+      "ldr q7, [x10, #0x50]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a452  // ummla v18.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49a  // ummla v26.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x60]\n"
+      "ldr q6, [x10, #0x60]\n"
       ".inst 0x6e87a40e  // ummla v14.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a456  // ummla v22.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49e  // ummla v30.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q7, [x10, #0x70]\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e86a40b  // ummla v11.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a453  // ummla v19.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49b  // ummla v27.4s, v4.16b, v6.16b\n"
@@ -3149,128 +3137,128 @@ void a64_hybrid_u8u32_mmla_6x16 (
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49f  // ummla v31.4s, v4.16b, v7.16b\n"
       "bge 206b\n"
-      "cbz x26, 212f\n"
       "207:"  // Height 6: Multiply loop: Skip odd blocks
-      "tbz x26, #2, 209f\n"
-      "ldr s1, [x25], #0x4\n"
-      "ldr s2, [x24], #0x4\n"
-      "ldr s3, [x23], #0x4\n"
-      "ldr s4, [x22], #0x4\n"
-      "ldr s5, [x21], #0x4\n"
-      "ldr s6, [x20], #0x4\n"
-      "tbz x26, #1, 208f\n"
-      "ld1 { v1.h }[2], [x25], #0x2\n"
-      "ld1 { v2.h }[2], [x24], #0x2\n"
-      "ld1 { v3.h }[2], [x23], #0x2\n"
-      "ld1 { v4.h }[2], [x22], #0x2\n"
-      "ld1 { v5.h }[2], [x21], #0x2\n"
-      "ld1 { v6.h }[2], [x20], #0x2\n"
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[6], [x25]\n"
-      "ld1 { v2.b }[6], [x24]\n"
-      "ld1 { v3.b }[6], [x23]\n"
-      "ld1 { v4.b }[6], [x22]\n"
-      "ld1 { v5.b }[6], [x21]\n"
-      "ld1 { v6.b }[6], [x20]\n"
+      "cbz x27, 212f\n"
+      "tbz x27, #2, 209f\n"
+      "ldr s1, [x26], #0x4\n"
+      "ldr s2, [x25], #0x4\n"
+      "ldr s3, [x24], #0x4\n"
+      "ldr s4, [x23], #0x4\n"
+      "ldr s5, [x22], #0x4\n"
+      "ldr s6, [x21], #0x4\n"
+      "tbz x27, #1, 208f\n"
+      "ld1 { v1.h }[2], [x26], #0x2\n"
+      "ld1 { v2.h }[2], [x25], #0x2\n"
+      "ld1 { v3.h }[2], [x24], #0x2\n"
+      "ld1 { v4.h }[2], [x23], #0x2\n"
+      "ld1 { v5.h }[2], [x22], #0x2\n"
+      "ld1 { v6.h }[2], [x21], #0x2\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[6], [x26]\n"
+      "ld1 { v2.b }[6], [x25]\n"
+      "ld1 { v3.b }[6], [x24]\n"
+      "ld1 { v4.b }[6], [x23]\n"
+      "ld1 { v5.b }[6], [x22]\n"
+      "ld1 { v6.b }[6], [x21]\n"
       "b 211f\n"
       "208:"  // Height 6: Multiply loop: Ragged operand read: partial_1_4
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[4], [x25]\n"
-      "ld1 { v2.b }[4], [x24]\n"
-      "ld1 { v3.b }[4], [x23]\n"
-      "ld1 { v4.b }[4], [x22]\n"
-      "ld1 { v5.b }[4], [x21]\n"
-      "ld1 { v6.b }[4], [x20]\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[4], [x26]\n"
+      "ld1 { v2.b }[4], [x25]\n"
+      "ld1 { v3.b }[4], [x24]\n"
+      "ld1 { v4.b }[4], [x23]\n"
+      "ld1 { v5.b }[4], [x22]\n"
+      "ld1 { v6.b }[4], [x21]\n"
       "b 211f\n"
       "209:"  // Height 6: Multiply loop: Ragged operand read: partial_2_0
-      "tbz x26, #1, 210f\n"
-      "ldr h1, [x25], #0x2\n"
-      "ldr h2, [x24], #0x2\n"
-      "ldr h3, [x23], #0x2\n"
-      "ldr h4, [x22], #0x2\n"
-      "ldr h5, [x21], #0x2\n"
-      "ldr h6, [x20], #0x2\n"
-      "tbz x26, #0, 211f\n"
-      "ld1 { v1.b }[2], [x25]\n"
-      "ld1 { v2.b }[2], [x24]\n"
-      "ld1 { v3.b }[2], [x23]\n"
-      "ld1 { v4.b }[2], [x22]\n"
-      "ld1 { v5.b }[2], [x21]\n"
-      "ld1 { v6.b }[2], [x20]\n"
+      "tbz x27, #1, 210f\n"
+      "ldr h1, [x26], #0x2\n"
+      "ldr h2, [x25], #0x2\n"
+      "ldr h3, [x24], #0x2\n"
+      "ldr h4, [x23], #0x2\n"
+      "ldr h5, [x22], #0x2\n"
+      "ldr h6, [x21], #0x2\n"
+      "tbz x27, #0, 211f\n"
+      "ld1 { v1.b }[2], [x26]\n"
+      "ld1 { v2.b }[2], [x25]\n"
+      "ld1 { v3.b }[2], [x24]\n"
+      "ld1 { v4.b }[2], [x23]\n"
+      "ld1 { v5.b }[2], [x22]\n"
+      "ld1 { v6.b }[2], [x21]\n"
       "b 211f\n"
       "210:"  // Height 6: Multiply loop: Ragged operand read: partial_1_0
-      "ldr b1, [x25, #0x0]\n"
-      "ldr b2, [x24, #0x0]\n"
-      "ldr b3, [x23, #0x0]\n"
-      "ldr b4, [x22, #0x0]\n"
-      "ldr b5, [x21, #0x0]\n"
-      "ldr b6, [x20, #0x0]\n"
+      "ldr b1, [x26, #0x0]\n"
+      "ldr b2, [x25, #0x0]\n"
+      "ldr b3, [x24, #0x0]\n"
+      "ldr b4, [x23, #0x0]\n"
+      "ldr b5, [x22, #0x0]\n"
+      "ldr b6, [x21, #0x0]\n"
       "211:"  // Height 6: Multiply loop: Ragged operand read: Done
+      "ldr q7, [x10, #0x0]\n"
       "trn1 v0.2d, v1.2d, v2.2d\n"
-      "ldr q7, [x9, #0x0]\n"
       "trn1 v2.2d, v3.2d, v4.2d\n"
-      "trn1 v4.2d, v5.2d, v6.2d\n"
-      "ldr q6, [x9, #0x10]\n"
       ".inst 0x6e87a408  // ummla v8.4s, v0.16b, v7.16b\n"
+      "trn1 v4.2d, v5.2d, v6.2d\n"
+      "ldr q6, [x10, #0x10]\n"
       ".inst 0x6e87a450  // ummla v16.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a498  // ummla v24.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x20]\n"
+      "ldr q7, [x10, #0x20]\n"
       ".inst 0x6e86a40c  // ummla v12.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49c  // ummla v28.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x30]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6e87a409  // ummla v9.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a451  // ummla v17.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a499  // ummla v25.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x40]\n"
+      "ldr q7, [x10, #0x40]\n"
       ".inst 0x6e86a40d  // ummla v13.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49d  // ummla v29.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x50]\n"
+      "ldr q6, [x10, #0x50]\n"
       ".inst 0x6e87a40a  // ummla v10.4s, v0.16b, v7.16b\n"
       ".inst 0x6e87a452  // ummla v18.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49a  // ummla v26.4s, v4.16b, v7.16b\n"
-      "ldr q7, [x9, #0x60]\n"
+      "ldr q7, [x10, #0x60]\n"
       ".inst 0x6e86a40e  // ummla v14.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49e  // ummla v30.4s, v4.16b, v6.16b\n"
-      "ldr q6, [x9, #0x70]\n"
-      "add x9, x9, #0x80\n"
+      "ldr q6, [x10, #0x70]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      "add x10, x10, #0x80\n"
       ".inst 0x6e87a453  // ummla v19.4s, v2.16b, v7.16b\n"
       ".inst 0x6e87a49b  // ummla v27.4s, v4.16b, v7.16b\n"
       ".inst 0x6e86a40f  // ummla v15.4s, v0.16b, v6.16b\n"
       ".inst 0x6e86a457  // ummla v23.4s, v2.16b, v6.16b\n"
       ".inst 0x6e86a49f  // ummla v31.4s, v4.16b, v6.16b\n"
       "212:"  // Height 6: Multiply loop: No odd multiplies
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 200b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 v7.2d, v8.2d, v12.2d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 v8.2d, v8.2d, v12.2d\n"
-      "prfm pstl1keep, [x28, #0x0]\n"
-      "cmp x10, #0x10\n"
       "uzp1 v12.2d, v9.2d, v13.2d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x11, #0x10\n"
       "uzp2 v9.2d, v9.2d, v13.2d\n"
-      "prfm pstl1keep, [x23, #0x0]\n"
       "uzp1 v13.2d, v10.2d, v14.2d\n"
-      "add x22, x23, x19, LSL #2\n"
       "uzp2 v10.2d, v10.2d, v14.2d\n"
-      "prfm pstl1keep, [x22, #0x0]\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 v14.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x21, #0x0]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "prfm pstl1keep, [x9, #0x0]\n"
+      "prfm pstl1keep, [x24, #0x0]\n"
       "uzp2 v11.2d, v11.2d, v15.2d\n"
-      "prfm pstl1keep, [x20, #0x0]\n"
-      "add x19, x20, x19, LSL #2\n"
       "uzp1 v15.2d, v16.2d, v20.2d\n"
-      "prfm pstl1keep, [x19, #0x0]\n"
+      "prfm pstl1keep, [x23, #0x0]\n"
+      "prfm pstl1keep, [x22, #0x0]\n"
       "uzp2 v16.2d, v16.2d, v20.2d\n"
       "uzp1 v20.2d, v17.2d, v21.2d\n"
+      "prfm pstl1keep, [x21, #0x0]\n"
+      "prfm pstl1keep, [x20, #0x0]\n"
       "uzp2 v17.2d, v17.2d, v21.2d\n"
       "uzp1 v21.2d, v18.2d, v22.2d\n"
       "uzp2 v18.2d, v18.2d, v22.2d\n"
@@ -3285,177 +3273,177 @@ void a64_hybrid_u8u32_mmla_6x16 (
       "uzp1 v30.2d, v27.2d, v31.2d\n"
       "uzp2 v27.2d, v27.2d, v31.2d\n"
       "bge 221f\n"
-      "tbz x10, #3, 216f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v12.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v9.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v20.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v17.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
-      "st1 { v28.4s }, [x20], #0x10\n"
-      "st1 { v24.4s }, [x19], #0x10\n"
-      "st1 { v25.4s }, [x19], #0x10\n"
-      "tbz x10, #2, 214f\n"
-      "st1 { v13.4s }, [x28], #0x10\n"
-      "st1 { v10.4s }, [x23], #0x10\n"
-      "st1 { v21.4s }, [x22], #0x10\n"
-      "st1 { v18.4s }, [x21], #0x10\n"
-      "st1 { v29.4s }, [x20], #0x10\n"
-      "st1 { v26.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 213f\n"
-      "str d14, [x28], #0x8\n"
-      "str d11, [x23], #0x8\n"
-      "str d22, [x22], #0x8\n"
-      "str d19, [x21], #0x8\n"
-      "str d30, [x20], #0x8\n"
-      "str d27, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v14.s }[2], [x28]\n"
-      "st1 { v11.s }[2], [x23]\n"
-      "st1 { v22.s }[2], [x22]\n"
-      "st1 { v19.s }[2], [x21]\n"
-      "st1 { v30.s }[2], [x20]\n"
-      "st1 { v27.s }[2], [x19]\n"
+      "tbz x11, #3, 216f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v12.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v9.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v20.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v17.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v28.4s }, [x21], #0x10\n"
+      "st1 { v24.4s }, [x20], #0x10\n"
+      "st1 { v25.4s }, [x20], #0x10\n"
+      "tbz x11, #2, 214f\n"
+      "st1 { v13.4s }, [x9], #0x10\n"
+      "st1 { v10.4s }, [x24], #0x10\n"
+      "st1 { v21.4s }, [x23], #0x10\n"
+      "st1 { v18.4s }, [x22], #0x10\n"
+      "st1 { v29.4s }, [x21], #0x10\n"
+      "st1 { v26.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 213f\n"
+      "str d14, [x9], #0x8\n"
+      "str d11, [x24], #0x8\n"
+      "str d22, [x23], #0x8\n"
+      "str d19, [x22], #0x8\n"
+      "str d30, [x21], #0x8\n"
+      "str d27, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v14.s }[2], [x9]\n"
+      "st1 { v11.s }[2], [x24]\n"
+      "st1 { v22.s }[2], [x23]\n"
+      "st1 { v19.s }[2], [x22]\n"
+      "st1 { v30.s }[2], [x21]\n"
+      "st1 { v27.s }[2], [x20]\n"
       "b 220f\n"
       "213:"  // Height 6: Partial direct writeback: partial_1_12
-      "tbz x10, #0, 220f\n"
-      "str s14, [x28, #0x0]\n"
-      "str s11, [x23, #0x0]\n"
-      "str s22, [x22, #0x0]\n"
-      "str s19, [x21, #0x0]\n"
-      "str s30, [x20, #0x0]\n"
-      "str s27, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s14, [x9, #0x0]\n"
+      "str s11, [x24, #0x0]\n"
+      "str s22, [x23, #0x0]\n"
+      "str s19, [x22, #0x0]\n"
+      "str s30, [x21, #0x0]\n"
+      "str s27, [x20, #0x0]\n"
       "b 220f\n"
       "214:"  // Height 6: Partial direct writeback: partial_2_8
-      "tbz x10, #1, 215f\n"
-      "str d13, [x28], #0x8\n"
-      "str d10, [x23], #0x8\n"
-      "str d21, [x22], #0x8\n"
-      "str d18, [x21], #0x8\n"
-      "str d29, [x20], #0x8\n"
-      "str d26, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v13.s }[2], [x28]\n"
-      "st1 { v10.s }[2], [x23]\n"
-      "st1 { v21.s }[2], [x22]\n"
-      "st1 { v18.s }[2], [x21]\n"
-      "st1 { v29.s }[2], [x20]\n"
-      "st1 { v26.s }[2], [x19]\n"
+      "tbz x11, #1, 215f\n"
+      "str d13, [x9], #0x8\n"
+      "str d10, [x24], #0x8\n"
+      "str d21, [x23], #0x8\n"
+      "str d18, [x22], #0x8\n"
+      "str d29, [x21], #0x8\n"
+      "str d26, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v13.s }[2], [x9]\n"
+      "st1 { v10.s }[2], [x24]\n"
+      "st1 { v21.s }[2], [x23]\n"
+      "st1 { v18.s }[2], [x22]\n"
+      "st1 { v29.s }[2], [x21]\n"
+      "st1 { v26.s }[2], [x20]\n"
       "b 220f\n"
       "215:"  // Height 6: Partial direct writeback: partial_1_8
-      "tbz x10, #0, 220f\n"
-      "str s13, [x28, #0x0]\n"
-      "str s10, [x23, #0x0]\n"
-      "str s21, [x22, #0x0]\n"
-      "str s18, [x21, #0x0]\n"
-      "str s29, [x20, #0x0]\n"
-      "str s26, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s13, [x9, #0x0]\n"
+      "str s10, [x24, #0x0]\n"
+      "str s21, [x23, #0x0]\n"
+      "str s18, [x22, #0x0]\n"
+      "str s29, [x21, #0x0]\n"
+      "str s26, [x20, #0x0]\n"
       "b 220f\n"
       "216:"  // Height 6: Partial direct writeback: partial_4_0
-      "tbz x10, #2, 218f\n"
-      "st1 { v7.4s }, [x28], #0x10\n"
-      "st1 { v8.4s }, [x23], #0x10\n"
-      "st1 { v15.4s }, [x22], #0x10\n"
-      "st1 { v16.4s }, [x21], #0x10\n"
-      "st1 { v23.4s }, [x20], #0x10\n"
-      "st1 { v24.4s }, [x19], #0x10\n"
-      "tbz x10, #1, 217f\n"
-      "str d12, [x28], #0x8\n"
-      "str d9, [x23], #0x8\n"
-      "str d20, [x22], #0x8\n"
-      "str d17, [x21], #0x8\n"
-      "str d28, [x20], #0x8\n"
-      "str d25, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v12.s }[2], [x28]\n"
-      "st1 { v9.s }[2], [x23]\n"
-      "st1 { v20.s }[2], [x22]\n"
-      "st1 { v17.s }[2], [x21]\n"
-      "st1 { v28.s }[2], [x20]\n"
-      "st1 { v25.s }[2], [x19]\n"
+      "tbz x11, #2, 218f\n"
+      "st1 { v7.4s }, [x9], #0x10\n"
+      "st1 { v8.4s }, [x24], #0x10\n"
+      "st1 { v15.4s }, [x23], #0x10\n"
+      "st1 { v16.4s }, [x22], #0x10\n"
+      "st1 { v23.4s }, [x21], #0x10\n"
+      "st1 { v24.4s }, [x20], #0x10\n"
+      "tbz x11, #1, 217f\n"
+      "str d12, [x9], #0x8\n"
+      "str d9, [x24], #0x8\n"
+      "str d20, [x23], #0x8\n"
+      "str d17, [x22], #0x8\n"
+      "str d28, [x21], #0x8\n"
+      "str d25, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v12.s }[2], [x9]\n"
+      "st1 { v9.s }[2], [x24]\n"
+      "st1 { v20.s }[2], [x23]\n"
+      "st1 { v17.s }[2], [x22]\n"
+      "st1 { v28.s }[2], [x21]\n"
+      "st1 { v25.s }[2], [x20]\n"
       "b 220f\n"
       "217:"  // Height 6: Partial direct writeback: partial_1_4
-      "tbz x10, #0, 220f\n"
-      "str s12, [x28, #0x0]\n"
-      "str s9, [x23, #0x0]\n"
-      "str s20, [x22, #0x0]\n"
-      "str s17, [x21, #0x0]\n"
-      "str s28, [x20, #0x0]\n"
-      "str s25, [x19, #0x0]\n"
+      "tbz x11, #0, 220f\n"
+      "str s12, [x9, #0x0]\n"
+      "str s9, [x24, #0x0]\n"
+      "str s20, [x23, #0x0]\n"
+      "str s17, [x22, #0x0]\n"
+      "str s28, [x21, #0x0]\n"
+      "str s25, [x20, #0x0]\n"
       "b 220f\n"
       "218:"  // Height 6: Partial direct writeback: partial_2_0
-      "tbz x10, #1, 219f\n"
-      "str d7, [x28], #0x8\n"
-      "str d8, [x23], #0x8\n"
-      "str d15, [x22], #0x8\n"
-      "str d16, [x21], #0x8\n"
-      "str d23, [x20], #0x8\n"
-      "str d24, [x19], #0x8\n"
-      "tbz x10, #0, 220f\n"
-      "st1 { v7.s }[2], [x28]\n"
-      "st1 { v8.s }[2], [x23]\n"
-      "st1 { v15.s }[2], [x22]\n"
-      "st1 { v16.s }[2], [x21]\n"
-      "st1 { v23.s }[2], [x20]\n"
-      "st1 { v24.s }[2], [x19]\n"
+      "tbz x11, #1, 219f\n"
+      "str d7, [x9], #0x8\n"
+      "str d8, [x24], #0x8\n"
+      "str d15, [x23], #0x8\n"
+      "str d16, [x22], #0x8\n"
+      "str d23, [x21], #0x8\n"
+      "str d24, [x20], #0x8\n"
+      "tbz x11, #0, 220f\n"
+      "st1 { v7.s }[2], [x9]\n"
+      "st1 { v8.s }[2], [x24]\n"
+      "st1 { v15.s }[2], [x23]\n"
+      "st1 { v16.s }[2], [x22]\n"
+      "st1 { v23.s }[2], [x21]\n"
+      "st1 { v24.s }[2], [x20]\n"
       "b 220f\n"
       "219:"  // Height 6: Partial direct writeback: partial_1_0
-      "str s7, [x28, #0x0]\n"
-      "str s8, [x23, #0x0]\n"
-      "str s15, [x22, #0x0]\n"
-      "str s16, [x21, #0x0]\n"
-      "str s23, [x20, #0x0]\n"
-      "str s24, [x19, #0x0]\n"
+      "str s7, [x9, #0x0]\n"
+      "str s8, [x24, #0x0]\n"
+      "str s15, [x23, #0x0]\n"
+      "str s16, [x22, #0x0]\n"
+      "str s23, [x21, #0x0]\n"
+      "str s24, [x20, #0x0]\n"
       "220:"  // Height 6: Partial direct writeback: Done
       "b 222f\n"
       "221:"  // Height 6: Full writeback
-      "str q7, [x28, #0x0]\n"
-      "str q12, [x28, #0x10]\n"
-      "str q13, [x28, #0x20]\n"
-      "str q14, [x28, #0x30]\n"
-      "add x28, x28, #0x40\n"
-      "str q8, [x23, #0x0]\n"
-      "str q9, [x23, #0x10]\n"
-      "str q10, [x23, #0x20]\n"
-      "str q11, [x23, #0x30]\n"
-      "str q15, [x22, #0x0]\n"
-      "str q20, [x22, #0x10]\n"
-      "str q21, [x22, #0x20]\n"
-      "str q22, [x22, #0x30]\n"
-      "str q16, [x21, #0x0]\n"
-      "str q17, [x21, #0x10]\n"
-      "str q18, [x21, #0x20]\n"
-      "str q19, [x21, #0x30]\n"
-      "str q23, [x20, #0x0]\n"
-      "str q28, [x20, #0x10]\n"
-      "str q29, [x20, #0x20]\n"
-      "str q30, [x20, #0x30]\n"
-      "str q24, [x19, #0x0]\n"
-      "str q25, [x19, #0x10]\n"
-      "str q26, [x19, #0x20]\n"
-      "str q27, [x19, #0x30]\n"
+      "str q7, [x9, #0x0]\n"
+      "str q12, [x9, #0x10]\n"
+      "str q13, [x9, #0x20]\n"
+      "str q14, [x9, #0x30]\n"
+      "add x9, x9, #0x40\n"
+      "str q8, [x24, #0x0]\n"
+      "str q9, [x24, #0x10]\n"
+      "str q10, [x24, #0x20]\n"
+      "str q11, [x24, #0x30]\n"
+      "str q15, [x23, #0x0]\n"
+      "str q20, [x23, #0x10]\n"
+      "str q21, [x23, #0x20]\n"
+      "str q22, [x23, #0x30]\n"
+      "str q16, [x22, #0x0]\n"
+      "str q17, [x22, #0x10]\n"
+      "str q18, [x22, #0x20]\n"
+      "str q19, [x22, #0x30]\n"
+      "str q23, [x21, #0x0]\n"
+      "str q28, [x21, #0x10]\n"
+      "str q29, [x21, #0x20]\n"
+      "str q30, [x21, #0x30]\n"
+      "str q24, [x20, #0x0]\n"
+      "str q25, [x20, #0x10]\n"
+      "str q26, [x20, #0x20]\n"
+      "str q27, [x20, #0x30]\n"
       "222:"  // Height 6: Writeback done
-      "subs x10, x10, #0x10\n"
+      "subs x11, x11, #0x10\n"
       "bgt 187b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 224f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 223f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "223:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "224:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
index 5689f89781..b3bde74635 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,108 +33,108 @@ void a64_interleaved_bf16fp32_dot_8x12(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const bfloat16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/2) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x22, #0x10]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "ldr q0, [%x[Apanel], #0x0]\n"
+      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.16b, #0x0\n"
+      "ldr q6, [x22, #0x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.16b, #0x0\n"
       "prfm pldl1keep, [%x[Apanel], #0x0]\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
-      "prfm pldl1keep, [x20, #0x0]\n"
+      "prfm pldl1keep, [x22, #0x0]\n"
       "movi v12.16b, #0x0\n"
       "movi v13.16b, #0x0\n"
-      "prfm pldl1keep, [x20, #0x40]\n"
+      "prfm pldl1keep, [x22, #0x40]\n"
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
       "prfm pldl1keep, [%x[Apanel], #0x40]\n"
       "movi v16.16b, #0x0\n"
       "movi v17.16b, #0x0\n"
-      "prfm pldl1keep, [x20, #0x80]\n"
+      "prfm pldl1keep, [x22, #0x80]\n"
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
       "movi v20.16b, #0x0\n"
       "movi v21.16b, #0x0\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
-      "ldr q4, [x20, #0x0]\n"
       "movi v24.16b, #0x0\n"
       "movi v25.16b, #0x0\n"
-      "ldr q5, [x20, #0x10]\n"
       "movi v26.16b, #0x0\n"
       "movi v27.16b, #0x0\n"
-      "ldr q6, [x20, #0x20]\n"
       "movi v28.16b, #0x0\n"
       "movi v29.16b, #0x0\n"
       "movi v30.16b, #0x0\n"
       "movi v31.16b, #0x0\n"
       "blt 4f\n"
       "3:"  // main loop head
+      "ldr q2, [%x[Apanel], #0x20]\n"
+      "ldr q3, [%x[Apanel], #0x30]\n"
       ".inst 0x4f40f088  // bfdot v8.4s, v4.8h, v0.h[0]\n"
       ".inst 0x4f60f08b  // bfdot v11.4s, v4.8h, v0.h[1]\n"
-      "ldr q2, [%x[Apanel], #0x20]\n"
       ".inst 0x4f40f88e  // bfdot v14.4s, v4.8h, v0.h[2]\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x4f60f891  // bfdot v17.4s, v4.8h, v0.h[3]\n"
-      "ldr q3, [%x[Apanel], #0x30]\n"
       ".inst 0x4f41f094  // bfdot v20.4s, v4.8h, v1.h[0]\n"
+      "cmp x20, #0x2\n"
       ".inst 0x4f61f097  // bfdot v23.4s, v4.8h, v1.h[1]\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x4f41f89a  // bfdot v26.4s, v4.8h, v1.h[2]\n"
+      "prfm pldl1keep, [%x[Apanel], #0x80]\n"
       ".inst 0x4f61f89d  // bfdot v29.4s, v4.8h, v1.h[3]\n"
-      "ldr q4, [x20, #0x30]\n"
+      "ldr q4, [x22, #0x30]\n"
       ".inst 0x4f40f0a9  // bfdot v9.4s, v5.8h, v0.h[0]\n"
       ".inst 0x4f60f0ac  // bfdot v12.4s, v5.8h, v0.h[1]\n"
-      "cmp x19, #0x2\n"
       ".inst 0x4f40f8af  // bfdot v15.4s, v5.8h, v0.h[2]\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x4f60f8b2  // bfdot v18.4s, v5.8h, v0.h[3]\n"
-      "prfm pldl1keep, [%x[Apanel], #0x80]\n"
       ".inst 0x4f41f0b5  // bfdot v21.4s, v5.8h, v1.h[0]\n"
+      "prfm pldl1keep, [x22, #0x100]\n"
       ".inst 0x4f61f0b8  // bfdot v24.4s, v5.8h, v1.h[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x4f41f8bb  // bfdot v27.4s, v5.8h, v1.h[2]\n"
+      "prfm pldl1keep, [x22, #0x140]\n"
       ".inst 0x4f61f8be  // bfdot v30.4s, v5.8h, v1.h[3]\n"
-      "ldr q5, [x20, #0x40]\n"
+      "ldr q5, [x22, #0x40]\n"
       ".inst 0x4f40f0ca  // bfdot v10.4s, v6.8h, v0.h[0]\n"
       ".inst 0x4f60f0cd  // bfdot v13.4s, v6.8h, v0.h[1]\n"
-      "prfm pldl1keep, [x20, #0x100]\n"
       ".inst 0x4f40f8d0  // bfdot v16.4s, v6.8h, v0.h[2]\n"
       ".inst 0x4f60f8d3  // bfdot v19.4s, v6.8h, v0.h[3]\n"
-      "prfm pldl1keep, [x20, #0x140]\n"
+      "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x4f41f0d6  // bfdot v22.4s, v6.8h, v1.h[0]\n"
       ".inst 0x4f61f0d9  // bfdot v25.4s, v6.8h, v1.h[1]\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
-      "ldr q6, [x20, #0x50]\n"
-      "add x20, x20, #0x60\n"
+      "ldr q6, [x22, #0x50]\n"
+      "ldr q1, [%x[Apanel], #0x10]\n"
+      "add x22, x22, #0x60\n"
       ".inst 0x4f42f088  // bfdot v8.4s, v4.8h, v2.h[0]\n"
       ".inst 0x4f62f08b  // bfdot v11.4s, v4.8h, v2.h[1]\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
       ".inst 0x4f42f88e  // bfdot v14.4s, v4.8h, v2.h[2]\n"
       ".inst 0x4f62f891  // bfdot v17.4s, v4.8h, v2.h[3]\n"
       ".inst 0x4f43f094  // bfdot v20.4s, v4.8h, v3.h[0]\n"
       ".inst 0x4f63f097  // bfdot v23.4s, v4.8h, v3.h[1]\n"
       ".inst 0x4f43f89a  // bfdot v26.4s, v4.8h, v3.h[2]\n"
       ".inst 0x4f63f89d  // bfdot v29.4s, v4.8h, v3.h[3]\n"
-      "ldr q4, [x20, #0x0]\n"
+      "ldr q4, [x22, #0x0]\n"
       ".inst 0x4f42f0a9  // bfdot v9.4s, v5.8h, v2.h[0]\n"
       ".inst 0x4f62f0ac  // bfdot v12.4s, v5.8h, v2.h[1]\n"
       ".inst 0x4f42f8af  // bfdot v15.4s, v5.8h, v2.h[2]\n"
@@ -143,7 +143,7 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f63f0b8  // bfdot v24.4s, v5.8h, v3.h[1]\n"
       ".inst 0x4f43f8bb  // bfdot v27.4s, v5.8h, v3.h[2]\n"
       ".inst 0x4f63f8be  // bfdot v30.4s, v5.8h, v3.h[3]\n"
-      "ldr q5, [x20, #0x10]\n"
+      "ldr q5, [x22, #0x10]\n"
       ".inst 0x4f42f0ca  // bfdot v10.4s, v6.8h, v2.h[0]\n"
       ".inst 0x4f62f0cd  // bfdot v13.4s, v6.8h, v2.h[1]\n"
       ".inst 0x4f42f8d0  // bfdot v16.4s, v6.8h, v2.h[2]\n"
@@ -152,13 +152,13 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f63f0d9  // bfdot v25.4s, v6.8h, v3.h[1]\n"
       ".inst 0x4f43f8dc  // bfdot v28.4s, v6.8h, v3.h[2]\n"
       ".inst 0x4f63f8df  // bfdot v31.4s, v6.8h, v3.h[3]\n"
-      "ldr q6, [x20, #0x20]\n"
+      "ldr q6, [x22, #0x20]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "add %x[Apanel], %x[Apanel], #0x20\n"
       ".inst 0x4f40f088  // bfdot v8.4s, v4.8h, v0.h[0]\n"
       ".inst 0x4f60f08b  // bfdot v11.4s, v4.8h, v0.h[1]\n"
-      "add x20, x20, #0x30\n"
+      "add x22, x22, #0x30\n"
       ".inst 0x4f40f88e  // bfdot v14.4s, v4.8h, v0.h[2]\n"
       ".inst 0x4f60f891  // bfdot v17.4s, v4.8h, v0.h[3]\n"
       ".inst 0x4f41f094  // bfdot v20.4s, v4.8h, v1.h[0]\n"
@@ -181,19 +181,19 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f61f0d9  // bfdot v25.4s, v6.8h, v1.h[1]\n"
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ldr q7, [x20, #0x0]\n"
-      "ldr q4, [x20, #0x10]\n"
+      "ldr q7, [x22, #0x0]\n"
+      "ldr q4, [x22, #0x10]\n"
       ".inst 0x4f40f0e8  // bfdot v8.4s, v7.8h, v0.h[0]\n"
-      "ldr q5, [x20, #0x20]\n"
+      "ldr q5, [x22, #0x20]\n"
       ".inst 0x4f60f0eb  // bfdot v11.4s, v7.8h, v0.h[1]\n"
       ".inst 0x4f40f8ee  // bfdot v14.4s, v7.8h, v0.h[2]\n"
-      "add x20, x20, #0x30\n"
       ".inst 0x4f60f8f1  // bfdot v17.4s, v7.8h, v0.h[3]\n"
       ".inst 0x4f41f0f4  // bfdot v20.4s, v7.8h, v1.h[0]\n"
+      "add x22, x22, #0x30\n"
       ".inst 0x4f61f0f7  // bfdot v23.4s, v7.8h, v1.h[1]\n"
       ".inst 0x4f41f8fa  // bfdot v26.4s, v7.8h, v1.h[2]\n"
       ".inst 0x4f61f8fd  // bfdot v29.4s, v7.8h, v1.h[3]\n"
@@ -214,7 +214,7 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f41f8bc  // bfdot v28.4s, v5.8h, v1.h[2]\n"
       ".inst 0x4f61f8bf  // bfdot v31.4s, v5.8h, v1.h[3]\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "str q8, [%x[Cpanel], #0x0]\n"
       "str q9, [%x[Cpanel], #0x10]\n"
       "str q10, [%x[Cpanel], #0x20]\n"
@@ -245,7 +245,7 @@ void a64_interleaved_bf16fp32_dot_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp
index 0235e91bfe..cba29bc572 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -33,34 +33,33 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const bfloat16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldp q4, q5, [x22], #0x20\n"
       "mov %x[Apanel], x21\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
       "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
-      "ld1 { v4.8h }, [x20], #0x10\n"
       "ld1 { v1.8h }, [%x[Apanel]], #0x10\n"
-      "cmp x19, #0x2\n"
       "movi v8.16b, #0x0\n"
+      "ld1 { v2.8h }, [%x[Apanel]], #0x10\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.16b, #0x0\n"
-      "ld1 { v5.8h }, [x20], #0x10\n"
       "movi v10.16b, #0x0\n"
       "movi v11.16b, #0x0\n"
-      "ld1 { v2.8h }, [%x[Apanel]], #0x10\n"
       "movi v12.16b, #0x0\n"
       "movi v13.16b, #0x0\n"
       "movi v14.16b, #0x0\n"
@@ -85,65 +84,63 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510(
       "3:"  // main loop head
       "ld1 { v3.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
-      "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
-      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
-      "ld1 { v6.8h }, [x20], #0x10\n"
+      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
+      "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e44ec30  // bfmmla v16.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec33  // bfmmla v19.4s, v1.8h, v5.8h\n"
       "ld1 { v1.8h }, [%x[Apanel]], #0x10\n"
-      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
-      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
-      "ld1 { v7.8h }, [x20], #0x10\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       "ld1 { v2.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
       "ld1 { v3.8h }, [%x[Apanel]], #0x10\n"
-      "ldp q4, q5, [x20], #0x20\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
-      ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
-      ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
+      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec55  // bfmmla v21.4s, v2.8h, v4.8h\n"
+      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7b  // bfmmla v27.4s, v3.8h, v4.8h\n"
-      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
-      "ld1 { v4.8h }, [x20], #0x10\n"
+      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
+      "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e46ec30  // bfmmla v16.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec33  // bfmmla v19.4s, v1.8h, v7.8h\n"
       "ld1 { v1.8h }, [%x[Apanel]], #0x10\n"
-      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
-      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
-      "ld1 { v5.8h }, [x20], #0x10\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec59  // bfmmla v25.4s, v2.8h, v7.8h\n"
       "ld1 { v2.8h }, [%x[Apanel]], #0x10\n"
@@ -153,66 +150,65 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510(
       "4:"  // main loop skip
       "ld1 { v3.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
       ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
-      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e44ec30  // bfmmla v16.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec33  // bfmmla v19.4s, v1.8h, v5.8h\n"
-      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
-      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
+      "ldp q6, q7, [x22], #0x20\n"
       "ld1 { v0.8h }, [%x[Apanel]], #0x10\n"
-      "ld1 { v6.8h }, [x20], #0x10\n"
-      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ld1 { v1.8h }, [%x[Apanel]], #0x10\n"
-      "ld1 { v7.8h }, [x20], #0x10\n"
-      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
+      ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ld1 { v2.8h }, [%x[Apanel]], #0x10\n"
       "ld1 { v3.8h }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
+      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec55  // bfmmla v21.4s, v2.8h, v4.8h\n"
+      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7b  // bfmmla v27.4s, v3.8h, v4.8h\n"
-      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
+      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       ".inst 0x6e46ec30  // bfmmla v16.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec33  // bfmmla v19.4s, v1.8h, v7.8h\n"
-      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
-      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec59  // bfmmla v25.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7c  // bfmmla v28.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7f  // bfmmla v31.4s, v3.8h, v7.8h\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -267,7 +263,7 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
index 94c72a31c9..2938639048 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,46 +33,46 @@ void a64_interleaved_bf16fp32_mmla_8x12(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const bfloat16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x22, #0x10]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "ldr q0, [%x[Apanel], #0x0]\n"
+      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.16b, #0x0\n"
+      "ldr q2, [%x[Apanel], #0x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.16b, #0x0\n"
-      "ldr q4, [x20, #0x0]\n"
       "movi v10.16b, #0x0\n"
+      "add x22, x22, #0x20\n"
       "movi v11.16b, #0x0\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
       "movi v12.16b, #0x0\n"
+      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v13.16b, #0x0\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v14.16b, #0x0\n"
       "movi v15.16b, #0x0\n"
-      "ldr q5, [x20, #0x10]\n"
       "movi v16.16b, #0x0\n"
       "movi v17.16b, #0x0\n"
-      "ldr q2, [%x[Apanel], #0x20]\n"
       "movi v18.16b, #0x0\n"
       "movi v19.16b, #0x0\n"
-      "add x20, x20, #0x20\n"
       "movi v20.16b, #0x0\n"
       "movi v21.16b, #0x0\n"
-      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v22.16b, #0x0\n"
       "movi v23.16b, #0x0\n"
       "movi v24.16b, #0x0\n"
@@ -86,149 +86,149 @@ void a64_interleaved_bf16fp32_mmla_8x12(
       "blt 4f\n"
       "3:"  // main loop head
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
+      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      "cmp x20, #0x2\n"
+      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
-      "ldr q6, [x20, #0x40]\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      "ldr q6, [x22, #0x40]\n"
+      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
-      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       "ldr q0, [%x[Apanel], #0x10]\n"
       ".inst 0x6e44ec30  // bfmmla v16.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec33  // bfmmla v19.4s, v1.8h, v5.8h\n"
       "ldr q1, [%x[Apanel], #0x20]\n"
-      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
-      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
-      "ldr q7, [x20, #0x50]\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       "ldr q2, [%x[Apanel], #0x30]\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
+      "ldr q4, [x22, #0x60]\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
       "ldr q3, [%x[Apanel], #0x40]\n"
+      "ldr q5, [x22, #0x70]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
-      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
-      "ldr q4, [x20, #0x60]\n"
       ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
-      "ldr q5, [x20, #0x70]\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
-      "cmp x19, #0x2\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
+      "ldr q6, [x22, #0x80]\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldr q6, [x20, #0x80]\n"
-      "ldr q7, [x20, #0x90]\n"
+      "ldr q7, [x22, #0x90]\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
+      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       ".inst 0x6e44ec55  // bfmmla v21.4s, v2.8h, v4.8h\n"
+      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7b  // bfmmla v27.4s, v3.8h, v4.8h\n"
-      "ldr q4, [x20, #0xa0]\n"
-      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
+      "ldr q4, [x22, #0xa0]\n"
+      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
+      "ldr q5, [x22, #0xb0]\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       "ldr q0, [%x[Apanel], #0x50]\n"
       ".inst 0x6e46ec30  // bfmmla v16.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec33  // bfmmla v19.4s, v1.8h, v7.8h\n"
       "ldr q1, [%x[Apanel], #0x60]\n"
-      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
-      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
-      "ldr q5, [x20, #0xb0]\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec59  // bfmmla v25.4s, v2.8h, v7.8h\n"
       "ldr q2, [%x[Apanel], #0x70]\n"
       ".inst 0x6e46ec7c  // bfmmla v28.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7f  // bfmmla v31.4s, v3.8h, v7.8h\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "add x20, x20, #0xc0\n"
+      "add x22, x22, #0xc0\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e44ec08  // bfmmla v8.4s, v0.8h, v4.8h\n"
-      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e45ec0b  // bfmmla v11.4s, v0.8h, v5.8h\n"
+      ".inst 0x6e44ec2e  // bfmmla v14.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec31  // bfmmla v17.4s, v1.8h, v5.8h\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x6e44ec54  // bfmmla v20.4s, v2.8h, v4.8h\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x6e45ec57  // bfmmla v23.4s, v2.8h, v5.8h\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x6e44ec7a  // bfmmla v26.4s, v3.8h, v4.8h\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x6e45ec7d  // bfmmla v29.4s, v3.8h, v5.8h\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e46ec09  // bfmmla v9.4s, v0.8h, v6.8h\n"
+      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
       ".inst 0x6e46ec2f  // bfmmla v15.4s, v1.8h, v6.8h\n"
+      "add x22, x22, #0x40\n"
+      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec55  // bfmmla v21.4s, v2.8h, v6.8h\n"
+      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7b  // bfmmla v27.4s, v3.8h, v6.8h\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x6e47ec0c  // bfmmla v12.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
       ".inst 0x6e44ec0a  // bfmmla v10.4s, v0.8h, v4.8h\n"
-      "add x20, x20, #0x40\n"
       ".inst 0x6e45ec0d  // bfmmla v13.4s, v0.8h, v5.8h\n"
-      ".inst 0x6e47ec32  // bfmmla v18.4s, v1.8h, v7.8h\n"
       ".inst 0x6e44ec30  // bfmmla v16.4s, v1.8h, v4.8h\n"
       ".inst 0x6e45ec33  // bfmmla v19.4s, v1.8h, v5.8h\n"
-      ".inst 0x6e47ec58  // bfmmla v24.4s, v2.8h, v7.8h\n"
-      ".inst 0x6e47ec7e  // bfmmla v30.4s, v3.8h, v7.8h\n"
       ".inst 0x6e44ec56  // bfmmla v22.4s, v2.8h, v4.8h\n"
       ".inst 0x6e45ec59  // bfmmla v25.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7c  // bfmmla v28.4s, v3.8h, v4.8h\n"
       ".inst 0x6e45ec7f  // bfmmla v31.4s, v3.8h, v5.8h\n"
-      "cbz x19, 5f\n"
-      "ldr q6, [x20, #0x0]\n"
+      "cbz x20, 5f\n"
+      "ldr q6, [x22, #0x0]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x6e46ec08  // bfmmla v8.4s, v0.8h, v6.8h\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q7, [x20, #0x10]\n"
-      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
+      "ldr q7, [x22, #0x10]\n"
+      ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
       "ldr q2, [%x[Apanel], #0x20]\n"
       "ldr q3, [%x[Apanel], #0x30]\n"
-      ".inst 0x6e47ec0b  // bfmmla v11.4s, v0.8h, v7.8h\n"
+      ".inst 0x6e46ec2e  // bfmmla v14.4s, v1.8h, v6.8h\n"
+      "ldr q4, [x22, #0x20]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e47ec31  // bfmmla v17.4s, v1.8h, v7.8h\n"
       ".inst 0x6e46ec54  // bfmmla v20.4s, v2.8h, v6.8h\n"
-      "ldr q4, [x20, #0x20]\n"
       ".inst 0x6e47ec57  // bfmmla v23.4s, v2.8h, v7.8h\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x6e46ec7a  // bfmmla v26.4s, v3.8h, v6.8h\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q6, [x22, #0x40]\n"
       ".inst 0x6e47ec7d  // bfmmla v29.4s, v3.8h, v7.8h\n"
-      "ldr q6, [x20, #0x40]\n"
-      "ldr q7, [x20, #0x50]\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x6e44ec09  // bfmmla v9.4s, v0.8h, v4.8h\n"
+      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
       ".inst 0x6e44ec2f  // bfmmla v15.4s, v1.8h, v4.8h\n"
-      "add x20, x20, #0x60\n"
+      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
+      "add x22, x22, #0x60\n"
       ".inst 0x6e44ec55  // bfmmla v21.4s, v2.8h, v4.8h\n"
+      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
       ".inst 0x6e44ec7b  // bfmmla v27.4s, v3.8h, v4.8h\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
-      ".inst 0x6e45ec0c  // bfmmla v12.4s, v0.8h, v5.8h\n"
+      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
       ".inst 0x6e46ec0a  // bfmmla v10.4s, v0.8h, v6.8h\n"
       ".inst 0x6e47ec0d  // bfmmla v13.4s, v0.8h, v7.8h\n"
-      ".inst 0x6e45ec32  // bfmmla v18.4s, v1.8h, v5.8h\n"
       ".inst 0x6e46ec30  // bfmmla v16.4s, v1.8h, v6.8h\n"
       ".inst 0x6e47ec33  // bfmmla v19.4s, v1.8h, v7.8h\n"
-      ".inst 0x6e45ec58  // bfmmla v24.4s, v2.8h, v5.8h\n"
-      ".inst 0x6e45ec7e  // bfmmla v30.4s, v3.8h, v5.8h\n"
       ".inst 0x6e46ec56  // bfmmla v22.4s, v2.8h, v6.8h\n"
       ".inst 0x6e47ec59  // bfmmla v25.4s, v2.8h, v7.8h\n"
       ".inst 0x6e46ec7c  // bfmmla v28.4s, v3.8h, v6.8h\n"
       ".inst 0x6e47ec7f  // bfmmla v31.4s, v3.8h, v7.8h\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -283,7 +283,7 @@ void a64_interleaved_bf16fp32_mmla_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp
index a4d8c0ace7..e46cb8a67a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -33,34 +33,33 @@ void a64_interleaved_s8s32_mmla_8x12_a510(
     int32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const int8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldp q4, q5, [x22], #0x20\n"
       "mov %x[Apanel], x21\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
       "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v4.16b }, [x20], #0x10\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      "cmp x19, #0x2\n"
       "movi v8.4s, #0x0\n"
+      "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.4s, #0x0\n"
-      "ld1 { v5.16b }, [x20], #0x10\n"
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
-      "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       "movi v12.4s, #0x0\n"
       "movi v13.4s, #0x0\n"
       "movi v14.4s, #0x0\n"
@@ -85,65 +84,63 @@ void a64_interleaved_s8s32_mmla_8x12_a510(
       "3:"  // main loop head
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e84a408  // smmla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x4e85a40b  // smmla v11.4s, v0.16b, v5.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a431  // smmla v17.4s, v1.16b, v5.16b\n"
       ".inst 0x4e84a454  // smmla v20.4s, v2.16b, v4.16b\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x4e85a457  // smmla v23.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47a  // smmla v26.4s, v3.16b, v4.16b\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x4e85a47d  // smmla v29.4s, v3.16b, v5.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
-      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47b  // smmla v27.4s, v3.16b, v6.16b\n"
-      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
-      "ld1 { v6.16b }, [x20], #0x10\n"
+      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
+      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e84a430  // smmla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
-      "ld1 { v7.16b }, [x20], #0x10\n"
       ".inst 0x4e84a456  // smmla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e84a47c  // smmla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x4e85a47f  // smmla v31.4s, v3.16b, v5.16b\n"
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
-      "ldp q4, q5, [x20], #0x20\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a431  // smmla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47a  // smmla v26.4s, v3.16b, v6.16b\n"
       ".inst 0x4e87a47d  // smmla v29.4s, v3.16b, v7.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x4e84a409  // smmla v9.4s, v0.16b, v4.16b\n"
       ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e84a42f  // smmla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x4e84a455  // smmla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47b  // smmla v27.4s, v3.16b, v4.16b\n"
-      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
-      "ld1 { v4.16b }, [x20], #0x10\n"
+      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
+      ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
+      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a433  // smmla v19.4s, v1.16b, v7.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
-      "ld1 { v5.16b }, [x20], #0x10\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
@@ -153,66 +150,65 @@ void a64_interleaved_s8s32_mmla_8x12_a510(
       "4:"  // main loop skip
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e84a408  // smmla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x4e85a40b  // smmla v11.4s, v0.16b, v5.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a431  // smmla v17.4s, v1.16b, v5.16b\n"
       ".inst 0x4e84a454  // smmla v20.4s, v2.16b, v4.16b\n"
       ".inst 0x4e85a457  // smmla v23.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47a  // smmla v26.4s, v3.16b, v4.16b\n"
       ".inst 0x4e85a47d  // smmla v29.4s, v3.16b, v5.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47b  // smmla v27.4s, v3.16b, v6.16b\n"
-      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
+      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
       ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e84a430  // smmla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
-      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x4e84a456  // smmla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47c  // smmla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x4e85a47f  // smmla v31.4s, v3.16b, v5.16b\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
+      "ldp q6, q7, [x22], #0x20\n"
       "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v6.16b }, [x20], #0x10\n"
-      ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v7.16b }, [x20], #0x10\n"
-      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a431  // smmla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47a  // smmla v26.4s, v3.16b, v6.16b\n"
       ".inst 0x4e87a47d  // smmla v29.4s, v3.16b, v7.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x4e84a409  // smmla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x4e84a42f  // smmla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x4e84a455  // smmla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47b  // smmla v27.4s, v3.16b, v4.16b\n"
-      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a433  // smmla v19.4s, v1.16b, v7.16b\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47c  // smmla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x4e87a47f  // smmla v31.4s, v3.16b, v7.16b\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -267,7 +263,7 @@ void a64_interleaved_s8s32_mmla_8x12_a510(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
index 0c2722a1c2..fc20c2fc9d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,46 +33,46 @@ void a64_interleaved_s8s32_mmla_8x12(
     int32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const int8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x22, #0x10]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x4\n"
+      "ldr q0, [%x[Apanel], #0x0]\n"
+      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.4s, #0x0\n"
+      "ldr q2, [%x[Apanel], #0x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.4s, #0x0\n"
-      "ldr q4, [x20, #0x0]\n"
       "movi v10.4s, #0x0\n"
+      "add x22, x22, #0x20\n"
       "movi v11.4s, #0x0\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
       "movi v12.4s, #0x0\n"
+      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v13.4s, #0x0\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
-      "ldr q5, [x20, #0x10]\n"
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
-      "ldr q2, [%x[Apanel], #0x20]\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
-      "add x20, x20, #0x20\n"
       "movi v20.4s, #0x0\n"
       "movi v21.4s, #0x0\n"
-      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "movi v24.4s, #0x0\n"
@@ -86,220 +86,149 @@ void a64_interleaved_s8s32_mmla_8x12(
       "blt 4f\n"
       "3:"  // main loop head
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x4e84a408  // smmla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x4e85a40b  // smmla v11.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a431  // smmla v17.4s, v1.16b, v5.16b\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x4e84a454  // smmla v20.4s, v2.16b, v4.16b\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x4e85a457  // smmla v23.4s, v2.16b, v5.16b\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x4e84a47a  // smmla v26.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x4e85a47d  // smmla v29.4s, v3.16b, v5.16b\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "cmp x20, #0x2\n"
+      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47b  // smmla v27.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x20, #0x40]\n"
-      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
+      "ldr q6, [x22, #0x40]\n"
+      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
-      "sub x19, x19, #0x4\n"
       ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       "ldr q0, [%x[Apanel], #0x10]\n"
       ".inst 0x4e84a430  // smmla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
       "ldr q1, [%x[Apanel], #0x20]\n"
-      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x20, #0x50]\n"
       ".inst 0x4e84a456  // smmla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
       "ldr q2, [%x[Apanel], #0x30]\n"
       ".inst 0x4e84a47c  // smmla v28.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x60]\n"
       ".inst 0x4e85a47f  // smmla v31.4s, v3.16b, v5.16b\n"
       "ldr q3, [%x[Apanel], #0x40]\n"
+      "ldr q5, [x22, #0x70]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q4, [x20, #0x60]\n"
       ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a431  // smmla v17.4s, v1.16b, v7.16b\n"
-      "ldr q5, [x20, #0x70]\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
-      "cmp x19, #0x4\n"
       ".inst 0x4e86a47a  // smmla v26.4s, v3.16b, v6.16b\n"
+      "ldr q6, [x22, #0x80]\n"
       ".inst 0x4e87a47d  // smmla v29.4s, v3.16b, v7.16b\n"
-      "ldr q6, [x20, #0x80]\n"
-      "ldr q7, [x20, #0x90]\n"
+      "ldr q7, [x22, #0x90]\n"
       ".inst 0x4e84a409  // smmla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x4e84a42f  // smmla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x4e84a455  // smmla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47b  // smmla v27.4s, v3.16b, v4.16b\n"
-      "ldr q4, [x20, #0xa0]\n"
-      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
+      "ldr q4, [x22, #0xa0]\n"
+      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
+      "ldr q5, [x22, #0xb0]\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       "ldr q0, [%x[Apanel], #0x50]\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a433  // smmla v19.4s, v1.16b, v7.16b\n"
       "ldr q1, [%x[Apanel], #0x60]\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
-      "ldr q5, [x20, #0xb0]\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
       "ldr q2, [%x[Apanel], #0x70]\n"
       ".inst 0x4e86a47c  // smmla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x4e87a47f  // smmla v31.4s, v3.16b, v7.16b\n"
-      "ldr q3, [%x[Apanel], #0x80]\n"
-      ".inst 0x4e84a408  // smmla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
-      "ldr q6, [x20, #0xc0]\n"
-      ".inst 0x4e85a40b  // smmla v11.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e85a431  // smmla v17.4s, v1.16b, v5.16b\n"
-      "ldr q7, [x20, #0xd0]\n"
-      ".inst 0x4e84a454  // smmla v20.4s, v2.16b, v4.16b\n"
-      ".inst 0x4e85a457  // smmla v23.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e84a47a  // smmla v26.4s, v3.16b, v4.16b\n"
-      ".inst 0x4e85a47d  // smmla v29.4s, v3.16b, v5.16b\n"
-      "ldr q4, [x20, #0xe0]\n"
-      "ldr q5, [x20, #0xf0]\n"
-      ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
-      ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
-      ".inst 0x4e86a47b  // smmla v27.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x20, #0x100]\n"
-      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
-      "ldr q0, [%x[Apanel], #0x90]\n"
-      ".inst 0x4e84a430  // smmla v16.4s, v1.16b, v4.16b\n"
-      ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
-      "ldr q1, [%x[Apanel], #0xa0]\n"
-      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x20, #0x110]\n"
-      ".inst 0x4e84a456  // smmla v22.4s, v2.16b, v4.16b\n"
-      ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
-      "ldr q2, [%x[Apanel], #0xb0]\n"
-      ".inst 0x4e84a47c  // smmla v28.4s, v3.16b, v4.16b\n"
-      ".inst 0x4e85a47f  // smmla v31.4s, v3.16b, v5.16b\n"
-      "ldr q3, [%x[Apanel], #0xc0]\n"
-      ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
-      "ldr q4, [x20, #0x120]\n"
-      ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e87a431  // smmla v17.4s, v1.16b, v7.16b\n"
-      "ldr q5, [x20, #0x130]\n"
-      ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e86a47a  // smmla v26.4s, v3.16b, v6.16b\n"
-      ".inst 0x4e87a47d  // smmla v29.4s, v3.16b, v7.16b\n"
-      "ldr q6, [x20, #0x140]\n"
-      "ldr q7, [x20, #0x150]\n"
-      ".inst 0x4e84a409  // smmla v9.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42f  // smmla v15.4s, v1.16b, v4.16b\n"
-      ".inst 0x4e84a455  // smmla v21.4s, v2.16b, v4.16b\n"
-      ".inst 0x4e84a47b  // smmla v27.4s, v3.16b, v4.16b\n"
-      "ldr q4, [x20, #0x160]\n"
-      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
-      "ldr q0, [%x[Apanel], #0xd0]\n"
-      ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
-      ".inst 0x4e87a433  // smmla v19.4s, v1.16b, v7.16b\n"
-      "ldr q1, [%x[Apanel], #0xe0]\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
-      "ldr q5, [x20, #0x170]\n"
-      ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
-      ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
-      "ldr q2, [%x[Apanel], #0xf0]\n"
-      ".inst 0x4e86a47c  // smmla v28.4s, v3.16b, v6.16b\n"
-      ".inst 0x4e87a47f  // smmla v31.4s, v3.16b, v7.16b\n"
-      "add %x[Apanel], %x[Apanel], #0x100\n"
-      "add x20, x20, #0x180\n"
+      "add %x[Apanel], %x[Apanel], #0x80\n"
+      "add x22, x22, #0xc0\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x4e84a408  // smmla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x4e85a40b  // smmla v11.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e84a42e  // smmla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a431  // smmla v17.4s, v1.16b, v5.16b\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x4e84a454  // smmla v20.4s, v2.16b, v4.16b\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x4e85a457  // smmla v23.4s, v2.16b, v5.16b\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x4e84a47a  // smmla v26.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x4e85a47d  // smmla v29.4s, v3.16b, v5.16b\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x4e86a409  // smmla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x4e86a42f  // smmla v15.4s, v1.16b, v6.16b\n"
+      "add x22, x22, #0x40\n"
+      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a455  // smmla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47b  // smmla v27.4s, v3.16b, v6.16b\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x4e87a40c  // smmla v12.4s, v0.16b, v7.16b\n"
+      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x4e84a40a  // smmla v10.4s, v0.16b, v4.16b\n"
-      "add x20, x20, #0x40\n"
       ".inst 0x4e85a40d  // smmla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x4e87a432  // smmla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x4e84a430  // smmla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x4e85a433  // smmla v19.4s, v1.16b, v5.16b\n"
-      ".inst 0x4e87a458  // smmla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x4e87a47e  // smmla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x4e84a456  // smmla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x4e85a459  // smmla v25.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47c  // smmla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x4e85a47f  // smmla v31.4s, v3.16b, v5.16b\n"
-      "cbz x19, 6f\n"
-      "5:"  // odd loop
-      "ldr q6, [x20, #0x0]\n"
+      "cbz x20, 5f\n"
+      "ldr q6, [x22, #0x0]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x4e86a408  // smmla v8.4s, v0.16b, v6.16b\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q7, [x20, #0x10]\n"
-      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      "ldr q7, [x22, #0x10]\n"
+      ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
       "ldr q2, [%x[Apanel], #0x20]\n"
       "ldr q3, [%x[Apanel], #0x30]\n"
-      ".inst 0x4e87a40b  // smmla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x4e86a42e  // smmla v14.4s, v1.16b, v6.16b\n"
+      "ldr q4, [x22, #0x20]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x4e87a431  // smmla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x4e86a454  // smmla v20.4s, v2.16b, v6.16b\n"
-      "ldr q4, [x20, #0x20]\n"
       ".inst 0x4e87a457  // smmla v23.4s, v2.16b, v7.16b\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x4e86a47a  // smmla v26.4s, v3.16b, v6.16b\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q6, [x22, #0x40]\n"
       ".inst 0x4e87a47d  // smmla v29.4s, v3.16b, v7.16b\n"
-      "ldr q6, [x20, #0x40]\n"
-      "ldr q7, [x20, #0x50]\n"
-      "subs x19, x19, #0x1\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x4e84a409  // smmla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x4e84a42f  // smmla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
+      "add x22, x22, #0x60\n"
       ".inst 0x4e84a455  // smmla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x4e84a47b  // smmla v27.4s, v3.16b, v4.16b\n"
-      "add x20, x20, #0x60\n"
-      ".inst 0x4e85a40c  // smmla v12.4s, v0.16b, v5.16b\n"
+      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x4e86a40a  // smmla v10.4s, v0.16b, v6.16b\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x4e87a40d  // smmla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x4e85a432  // smmla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x4e86a430  // smmla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x4e87a433  // smmla v19.4s, v1.16b, v7.16b\n"
-      ".inst 0x4e85a458  // smmla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x4e85a47e  // smmla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x4e86a456  // smmla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x4e87a459  // smmla v25.4s, v2.16b, v7.16b\n"
       ".inst 0x4e86a47c  // smmla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x4e87a47f  // smmla v31.4s, v3.16b, v7.16b\n"
-      "bne 5b\n"
-      "6:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "5:"  // multiply loop done
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -354,7 +283,7 @@ void a64_interleaved_s8s32_mmla_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp
index 3fe1a9bd04..83301d80bb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __aarch64__
 
@@ -33,34 +33,33 @@ void a64_interleaved_u8u32_mmla_8x12_a510(
     uint32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const uint8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldp q4, q5, [x22], #0x20\n"
       "mov %x[Apanel], x21\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
       "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v4.16b }, [x20], #0x10\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      "cmp x19, #0x2\n"
       "movi v8.4s, #0x0\n"
+      "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.4s, #0x0\n"
-      "ld1 { v5.16b }, [x20], #0x10\n"
       "movi v10.4s, #0x0\n"
       "movi v11.4s, #0x0\n"
-      "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       "movi v12.4s, #0x0\n"
       "movi v13.4s, #0x0\n"
       "movi v14.4s, #0x0\n"
@@ -85,65 +84,63 @@ void a64_interleaved_u8u32_mmla_8x12_a510(
       "3:"  // main loop head
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e84a408  // ummla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x6e85a40b  // ummla v11.4s, v0.16b, v5.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a431  // ummla v17.4s, v1.16b, v5.16b\n"
       ".inst 0x6e84a454  // ummla v20.4s, v2.16b, v4.16b\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6e85a457  // ummla v23.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47a  // ummla v26.4s, v3.16b, v4.16b\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x6e85a47d  // ummla v29.4s, v3.16b, v5.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
-      ".inst 0x6e84a40a  // ummla v10.4s, v0.16b, v4.16b\n"
-      ".inst 0x6e85a40d  // ummla v13.4s, v0.16b, v5.16b\n"
-      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47b  // ummla v27.4s, v3.16b, v6.16b\n"
-      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
-      "ld1 { v6.16b }, [x20], #0x10\n"
+      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e84a40a  // ummla v10.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e85a40d  // ummla v13.4s, v0.16b, v5.16b\n"
+      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e84a430  // ummla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a433  // ummla v19.4s, v1.16b, v5.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
-      "ld1 { v7.16b }, [x20], #0x10\n"
       ".inst 0x6e84a456  // ummla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e84a47c  // ummla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x6e85a47f  // ummla v31.4s, v3.16b, v5.16b\n"
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
-      "ldp q4, q5, [x20], #0x20\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
-      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a431  // ummla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47a  // ummla v26.4s, v3.16b, v6.16b\n"
       ".inst 0x6e87a47d  // ummla v29.4s, v3.16b, v7.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x6e84a409  // ummla v9.4s, v0.16b, v4.16b\n"
       ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
-      ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e84a42f  // ummla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x6e84a455  // ummla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47b  // ummla v27.4s, v3.16b, v4.16b\n"
-      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
-      "ld1 { v4.16b }, [x20], #0x10\n"
+      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
+      ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
+      "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a433  // ummla v19.4s, v1.16b, v7.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
-      "ld1 { v5.16b }, [x20], #0x10\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
@@ -153,66 +150,65 @@ void a64_interleaved_u8u32_mmla_8x12_a510(
       "4:"  // main loop skip
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e84a408  // ummla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
       ".inst 0x6e85a40b  // ummla v11.4s, v0.16b, v5.16b\n"
+      "ldp q6, q7, [x22], #0x20\n"
+      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a431  // ummla v17.4s, v1.16b, v5.16b\n"
       ".inst 0x6e84a454  // ummla v20.4s, v2.16b, v4.16b\n"
       ".inst 0x6e85a457  // ummla v23.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47a  // ummla v26.4s, v3.16b, v4.16b\n"
       ".inst 0x6e85a47d  // ummla v29.4s, v3.16b, v5.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47b  // ummla v27.4s, v3.16b, v6.16b\n"
-      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
+      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x6e84a40a  // ummla v10.4s, v0.16b, v4.16b\n"
       ".inst 0x6e85a40d  // ummla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e84a430  // ummla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a433  // ummla v19.4s, v1.16b, v5.16b\n"
-      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x6e84a456  // ummla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47c  // ummla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x6e85a47f  // ummla v31.4s, v3.16b, v5.16b\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
+      "ldp q6, q7, [x22], #0x20\n"
       "ld1 { v0.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v6.16b }, [x20], #0x10\n"
-      ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
       "ld1 { v1.16b }, [%x[Apanel]], #0x10\n"
-      "ld1 { v7.16b }, [x20], #0x10\n"
-      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
+      ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
       "ld1 { v2.16b }, [%x[Apanel]], #0x10\n"
       "ld1 { v3.16b }, [%x[Apanel]], #0x10\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
-      "ldp q4, q5, [x20], #0x20\n"
+      "ldp q4, q5, [x22], #0x20\n"
+      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a431  // ummla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47a  // ummla v26.4s, v3.16b, v6.16b\n"
       ".inst 0x6e87a47d  // ummla v29.4s, v3.16b, v7.16b\n"
-      "ldp q6, q7, [x20], #0x20\n"
+      "ldp q6, q7, [x22], #0x20\n"
       ".inst 0x6e84a409  // ummla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x6e84a42f  // ummla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x6e84a455  // ummla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47b  // ummla v27.4s, v3.16b, v4.16b\n"
-      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a433  // ummla v19.4s, v1.16b, v7.16b\n"
-      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47c  // ummla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x6e87a47f  // ummla v31.4s, v3.16b, v7.16b\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -267,7 +263,7 @@ void a64_interleaved_u8u32_mmla_8x12_a510(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
index e67d17e49a..c5342197c1 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020 Arm Limited.
+ * Copyright (c) 2019-2020, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,46 +33,46 @@ void a64_interleaved_u8u32_mmla_8x12(
     uint32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const uint8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
 
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr q4, [x22, #0x0]\n"
+      "ldr q5, [x22, #0x10]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "ldr q0, [%x[Apanel], #0x0]\n"
+      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v8.4s, #0x0\n"
+      "ldr q2, [%x[Apanel], #0x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "movi v9.4s, #0x0\n"
-      "ldr q4, [x20, #0x0]\n"
       "movi v10.4s, #0x0\n"
+      "add x22, x22, #0x20\n"
       "movi v11.4s, #0x0\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
       "movi v12.4s, #0x0\n"
+      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v13.4s, #0x0\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
       "movi v14.4s, #0x0\n"
       "movi v15.4s, #0x0\n"
-      "ldr q5, [x20, #0x10]\n"
       "movi v16.4s, #0x0\n"
       "movi v17.4s, #0x0\n"
-      "ldr q2, [%x[Apanel], #0x20]\n"
       "movi v18.4s, #0x0\n"
       "movi v19.4s, #0x0\n"
-      "add x20, x20, #0x20\n"
       "movi v20.4s, #0x0\n"
       "movi v21.4s, #0x0\n"
-      "add %x[Apanel], %x[Apanel], #0x30\n"
       "movi v22.4s, #0x0\n"
       "movi v23.4s, #0x0\n"
       "movi v24.4s, #0x0\n"
@@ -86,149 +86,149 @@ void a64_interleaved_u8u32_mmla_8x12(
       "blt 4f\n"
       "3:"  // main loop head
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e84a408  // ummla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e85a40b  // ummla v11.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a431  // ummla v17.4s, v1.16b, v5.16b\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x6e84a454  // ummla v20.4s, v2.16b, v4.16b\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6e85a457  // ummla v23.4s, v2.16b, v5.16b\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x6e84a47a  // ummla v26.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x6e85a47d  // ummla v29.4s, v3.16b, v5.16b\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      "cmp x20, #0x2\n"
+      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47b  // ummla v27.4s, v3.16b, v6.16b\n"
-      "ldr q6, [x20, #0x40]\n"
-      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
+      "ldr q6, [x22, #0x40]\n"
+      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x6e84a40a  // ummla v10.4s, v0.16b, v4.16b\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x6e85a40d  // ummla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       "ldr q0, [%x[Apanel], #0x10]\n"
       ".inst 0x6e84a430  // ummla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a433  // ummla v19.4s, v1.16b, v5.16b\n"
       "ldr q1, [%x[Apanel], #0x20]\n"
-      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
-      "ldr q7, [x20, #0x50]\n"
       ".inst 0x6e84a456  // ummla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
       "ldr q2, [%x[Apanel], #0x30]\n"
       ".inst 0x6e84a47c  // ummla v28.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x60]\n"
       ".inst 0x6e85a47f  // ummla v31.4s, v3.16b, v5.16b\n"
       "ldr q3, [%x[Apanel], #0x40]\n"
+      "ldr q5, [x22, #0x70]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
-      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
-      "ldr q4, [x20, #0x60]\n"
       ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a431  // ummla v17.4s, v1.16b, v7.16b\n"
-      "ldr q5, [x20, #0x70]\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
-      "cmp x19, #0x2\n"
       ".inst 0x6e86a47a  // ummla v26.4s, v3.16b, v6.16b\n"
+      "ldr q6, [x22, #0x80]\n"
       ".inst 0x6e87a47d  // ummla v29.4s, v3.16b, v7.16b\n"
-      "ldr q6, [x20, #0x80]\n"
-      "ldr q7, [x20, #0x90]\n"
+      "ldr q7, [x22, #0x90]\n"
       ".inst 0x6e84a409  // ummla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x6e84a42f  // ummla v15.4s, v1.16b, v4.16b\n"
+      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x6e84a455  // ummla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47b  // ummla v27.4s, v3.16b, v4.16b\n"
-      "ldr q4, [x20, #0xa0]\n"
-      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
+      "ldr q4, [x22, #0xa0]\n"
+      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
+      "ldr q5, [x22, #0xb0]\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       "ldr q0, [%x[Apanel], #0x50]\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a433  // ummla v19.4s, v1.16b, v7.16b\n"
       "ldr q1, [%x[Apanel], #0x60]\n"
-      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
-      "ldr q5, [x20, #0xb0]\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
       "ldr q2, [%x[Apanel], #0x70]\n"
       ".inst 0x6e86a47c  // ummla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x6e87a47f  // ummla v31.4s, v3.16b, v7.16b\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "add x20, x20, #0xc0\n"
+      "add x22, x22, #0xc0\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ldr q3, [%x[Apanel], #0x0]\n"
+      "ldr q6, [x22, #0x0]\n"
       ".inst 0x6e84a408  // ummla v8.4s, v0.16b, v4.16b\n"
-      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
+      "ldr q7, [x22, #0x10]\n"
       ".inst 0x6e85a40b  // ummla v11.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e84a42e  // ummla v14.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a431  // ummla v17.4s, v1.16b, v5.16b\n"
-      "ldr q6, [x20, #0x0]\n"
       ".inst 0x6e84a454  // ummla v20.4s, v2.16b, v4.16b\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x6e85a457  // ummla v23.4s, v2.16b, v5.16b\n"
-      "ldr q7, [x20, #0x10]\n"
       ".inst 0x6e84a47a  // ummla v26.4s, v3.16b, v4.16b\n"
+      "ldr q4, [x22, #0x20]\n"
       ".inst 0x6e85a47d  // ummla v29.4s, v3.16b, v5.16b\n"
-      "ldr q4, [x20, #0x20]\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e86a409  // ummla v9.4s, v0.16b, v6.16b\n"
+      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
       ".inst 0x6e86a42f  // ummla v15.4s, v1.16b, v6.16b\n"
+      "add x22, x22, #0x40\n"
+      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a455  // ummla v21.4s, v2.16b, v6.16b\n"
+      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47b  // ummla v27.4s, v3.16b, v6.16b\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x6e87a40c  // ummla v12.4s, v0.16b, v7.16b\n"
+      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x6e84a40a  // ummla v10.4s, v0.16b, v4.16b\n"
-      "add x20, x20, #0x40\n"
       ".inst 0x6e85a40d  // ummla v13.4s, v0.16b, v5.16b\n"
-      ".inst 0x6e87a432  // ummla v18.4s, v1.16b, v7.16b\n"
       ".inst 0x6e84a430  // ummla v16.4s, v1.16b, v4.16b\n"
       ".inst 0x6e85a433  // ummla v19.4s, v1.16b, v5.16b\n"
-      ".inst 0x6e87a458  // ummla v24.4s, v2.16b, v7.16b\n"
-      ".inst 0x6e87a47e  // ummla v30.4s, v3.16b, v7.16b\n"
       ".inst 0x6e84a456  // ummla v22.4s, v2.16b, v4.16b\n"
       ".inst 0x6e85a459  // ummla v25.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47c  // ummla v28.4s, v3.16b, v4.16b\n"
       ".inst 0x6e85a47f  // ummla v31.4s, v3.16b, v5.16b\n"
-      "cbz x19, 5f\n"
-      "ldr q6, [x20, #0x0]\n"
+      "cbz x20, 5f\n"
+      "ldr q6, [x22, #0x0]\n"
       "ldr q0, [%x[Apanel], #0x0]\n"
       ".inst 0x6e86a408  // ummla v8.4s, v0.16b, v6.16b\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
-      "ldr q7, [x20, #0x10]\n"
-      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
+      "ldr q7, [x22, #0x10]\n"
+      ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
       "ldr q2, [%x[Apanel], #0x20]\n"
       "ldr q3, [%x[Apanel], #0x30]\n"
-      ".inst 0x6e87a40b  // ummla v11.4s, v0.16b, v7.16b\n"
+      ".inst 0x6e86a42e  // ummla v14.4s, v1.16b, v6.16b\n"
+      "ldr q4, [x22, #0x20]\n"
+      "ldr q5, [x22, #0x30]\n"
       ".inst 0x6e87a431  // ummla v17.4s, v1.16b, v7.16b\n"
       ".inst 0x6e86a454  // ummla v20.4s, v2.16b, v6.16b\n"
-      "ldr q4, [x20, #0x20]\n"
       ".inst 0x6e87a457  // ummla v23.4s, v2.16b, v7.16b\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x6e86a47a  // ummla v26.4s, v3.16b, v6.16b\n"
-      "ldr q5, [x20, #0x30]\n"
+      "ldr q6, [x22, #0x40]\n"
       ".inst 0x6e87a47d  // ummla v29.4s, v3.16b, v7.16b\n"
-      "ldr q6, [x20, #0x40]\n"
-      "ldr q7, [x20, #0x50]\n"
+      "ldr q7, [x22, #0x50]\n"
       ".inst 0x6e84a409  // ummla v9.4s, v0.16b, v4.16b\n"
+      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
       ".inst 0x6e84a42f  // ummla v15.4s, v1.16b, v4.16b\n"
-      "add x20, x20, #0x60\n"
+      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
+      "add x22, x22, #0x60\n"
       ".inst 0x6e84a455  // ummla v21.4s, v2.16b, v4.16b\n"
+      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
       ".inst 0x6e84a47b  // ummla v27.4s, v3.16b, v4.16b\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
-      ".inst 0x6e85a40c  // ummla v12.4s, v0.16b, v5.16b\n"
+      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x6e86a40a  // ummla v10.4s, v0.16b, v6.16b\n"
       ".inst 0x6e87a40d  // ummla v13.4s, v0.16b, v7.16b\n"
-      ".inst 0x6e85a432  // ummla v18.4s, v1.16b, v5.16b\n"
       ".inst 0x6e86a430  // ummla v16.4s, v1.16b, v6.16b\n"
       ".inst 0x6e87a433  // ummla v19.4s, v1.16b, v7.16b\n"
-      ".inst 0x6e85a458  // ummla v24.4s, v2.16b, v5.16b\n"
-      ".inst 0x6e85a47e  // ummla v30.4s, v3.16b, v5.16b\n"
       ".inst 0x6e86a456  // ummla v22.4s, v2.16b, v6.16b\n"
       ".inst 0x6e87a459  // ummla v25.4s, v2.16b, v7.16b\n"
       ".inst 0x6e86a47c  // ummla v28.4s, v3.16b, v6.16b\n"
       ".inst 0x6e87a47f  // ummla v31.4s, v3.16b, v7.16b\n"
       "5:"  // multiply loop done
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "uzp1 v4.2d, v8.2d, v11.2d\n"
       "uzp2 v8.2d, v8.2d, v11.2d\n"
       "uzp1 v11.2d, v9.2d, v12.2d\n"
@@ -283,7 +283,7 @@ void a64_interleaved_u8u32_mmla_8x12(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+      : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp
index 26861fb931..520eeedfec 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -64,487 +64,487 @@ void sme2_gemv_bf16fp32_dot_16VL (
     __asm__ __volatile__(
       "ptrue p1.b\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
-      "cntw x27, ALL, MUL #4\n"
-      "add x26, %x[N], x27\n"
-      "sub x26, x26, #0x1\n"
-      "udiv x26, x26, x27\n"
-      "add x21, x26, #0x3\n"
-      "and x21, x21, #0xfffffffffffffffc\n"
-      "mul x21, x21, x27\n"
-      "mul x21, x21, %x[K]\n"
+      "cntw x28, ALL, MUL #4\n"
+      "add x27, %x[N], x28\n"
+      "sub x27, x27, #0x1\n"
+      "udiv x27, x27, x28\n"
+      "add x22, x27, #0x3\n"
+      "and x22, x22, #0xfffffffffffffffc\n"
+      "mul x22, x22, x28\n"
+      "mul x22, x22, %x[K]\n"
       "mov x9, #0x0\n"
-      "mov x25, %x[B_ptr]\n"
-      "mov x24, %x[output_ptr]\n"
+      "mov x26, %x[B_ptr]\n"
+      "mov x25, %x[output_ptr]\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
-      "lsl x21, x21, #0x1\n"
-      "mov x20, #0x1\n"
+      "lsl x22, x22, #0x1\n"
+      "mov x21, #0x1\n"
       "1:"  // RHS size check loop
-      "cmp x21, #0x200000\n"
+      "cmp x22, #0x200000\n"
       "blt 2f\n"
-      "tbnz x21, #0, 3f\n"
-      "lsr x21, x21, #0x1\n"
-      "lsl x20, x20, #0x1\n"
+      "tbnz x22, #0, 3f\n"
+      "lsr x22, x22, #0x1\n"
+      "lsl x21, x21, #0x1\n"
       "b 1b\n"
       "2:"  // RHS do prefetch
-      "lsl x19, x21, #0x26\n"
-      "sub x20, x20, #0x1\n"
-      "lsl x20, x20, #0x16\n"
-      "orr x21, x21, x19\n"
-      "orr x21, x21, x20\n"
-      ".inst 0xf8b54b3a  // rprfm pldonce, x21, [x25]\n"
+      "lsl x20, x22, #0x26\n"
+      "sub x21, x21, #0x1\n"
+      "lsl x21, x21, #0x16\n"
+      "orr x22, x22, x20\n"
+      "orr x22, x22, x21\n"
+      ".inst 0xf8b64b5a  // rprfm pldonce, x22, [x26]\n"
       "3:"  // RHS prefetch exit
-      "mov x23, %x[bias]\n"
+      "mov x24, %x[bias]\n"
       "4:"  // Column loop
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "bge 28f\n"
-      "cmp x26, #0x2\n"
+      "cmp x27, #0x2\n"
       "bgt 20f\n"
       "beq 12f\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x1\n"
-      "mov x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 5f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x1\n"
+      "mov x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 5f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
       "b 6f\n"
       "5:"  // Width 1: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "6:"  // Width 1: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 8f\n"
       "7:"  // Width 1: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x8\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x8\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
-      "cmp x20, #0x8\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      "cmp x21, #0x8\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 7b\n"
       "8:"  // Width 1: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x2\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x2\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "9:"  // Width 1: Multiply loop: multiply skip
       "tbz %x[flags], #1, 10f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "b 11f\n"
       "10:"  // Width 1: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "11:"  // Width 1: Output done
       "b 36f\n"
       "12:"  // Width 2
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x1\n"
-      "sub x19, %x[N], x27\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 13f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x1\n"
+      "sub x20, %x[N], x28\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 13f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
       "b 14f\n"
       "13:"  // Width 2: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "14:"  // Width 2: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 16f\n"
       "15:"  // Width 2: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x8\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x8\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      "cmp x20, #0x8\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x8\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 15b\n"
       "16:"  // Width 2: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x2\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x2\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "17:"  // Width 2: Multiply loop: multiply skip
       "tbz %x[flags], #1, 18f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
-      ".inst 0xa061c314  // st1w { z20.s-z23.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c334  // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "b 19f\n"
       "18:"  // Width 2: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c314  // st1w { z20.s-z23.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c334  // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "19:"  // Width 2: Output done
       "b 36f\n"
       "20:"  // Width 3
-      "mov x19, #0x2\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x1\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 21f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x2\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x1\n"
+      "msub x20, x28, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 21f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
-      ".inst 0xa042c6e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c704  // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042c82  // mova za.d[x9, #2], { z4.d-z7.d }\n"
       "b 22f\n"
       "21:"  // Width 3: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "22:"  // Width 3: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 24f\n"
       "23:"  // Width 3: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x8\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x8\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      "cmp x20, #0x8\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x8\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      ".inst 0xa042a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab29a  // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      ".inst 0xa042a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab71a  // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      ".inst 0xa042a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abb9a  // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abe1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 23b\n"
       "24:"  // Width 3: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x2\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x2\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      ".inst 0xa042a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab29a  // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      ".inst 0xa042a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab71a  // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      ".inst 0xa042a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abb9a  // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abe1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "25:"  // Width 3: Multiply loop: multiply skip
       "tbz %x[flags], #1, 26f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1a6c810  // fclamp { z16.s-z19.s }, z0.s, z6.s\n"
-      ".inst 0xa062c310  // st1w { z16.s-z19.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c330  // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "b 27f\n"
       "26:"  // Width 3: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c310  // st1w { z16.s-z19.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c330  // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "27:"  // Width 3: Output done
       "b 36f\n"
       "28:"  // Width 4
-      "mov x19, #0x3\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x1\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 29f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x3\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x1\n"
+      "msub x20, x28, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 29f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
-      ".inst 0xa042c6e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c704  // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042c82  // mova za.d[x9, #2], { z4.d-z7.d }\n"
-      ".inst 0xa043c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      ".inst 0xa043c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n"
       ".inst 0xc0042e03  // mova za.d[x9, #3], { z16.d-z19.d }\n"
-      "addvl x23, x23, #16\n"
+      "addvl x24, x24, #16\n"
       "b 30f\n"
       "29:"  // Width 4: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "30:"  // Width 4: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 32f\n"
       "31:"  // Width 4: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x8\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x8\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      "cmp x20, #0x8\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x8\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      ".inst 0xa042a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab29a  // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15ab21b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      ".inst 0xa042a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab71a  // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15ab61b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      ".inst 0xa042a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abb9a  // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n"
-      ".inst 0xa043a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aba9b  // bfdot za.s[x9, 3], { z20.h-z23.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abe1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15abe1b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 31b\n"
       "32:"  // Width 4: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x20\n"
-      "ld1rqh { z10.h }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x2\n"
-      ".inst 0xa040a721  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.h, XZR, x21\n"
+      "ld1rqh { z10.h }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x2\n"
+      ".inst 0xa040a741  // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15ab018  // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab099  // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n"
-      ".inst 0xa042a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab29a  // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15ab21b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab718  // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15ab619  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n"
-      ".inst 0xa042a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15ab71a  // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15ab61b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc15ab998  // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n"
-      ".inst 0xa041a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aba19  // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n"
-      ".inst 0xa042a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abb9a  // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n"
-      ".inst 0xa043a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aba9b  // bfdot za.s[x9, 3], { z20.h-z23.h }, z10.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc15abe18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa041a739  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a759  // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15abf19  // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15abe1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n"
-      ".inst 0xa043a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15abe1b  // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "33:"  // Width 4: Multiply loop: multiply skip
       "tbz %x[flags], #1, 34f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
       ".inst 0xc0062c78  // mova { z24.d-z27.d }, za.d[x9, #3]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1a6c810  // fclamp { z16.s-z19.s }, z0.s, z6.s\n"
-      ".inst 0xa062c710  // st1w { z16.s-z19.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c730  // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc1a6c818  // fclamp { z24.s-z27.s }, z0.s, z6.s\n"
-      ".inst 0xa063c318  // st1w { z24.s-z27.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c338  // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "b 35f\n"
       "34:"  // Width 4: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c710  // st1w { z16.s-z19.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c730  // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc0062c78  // mova { z24.d-z27.d }, za.d[x9, #3]\n"
-      ".inst 0xa063c318  // st1w { z24.s-z27.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c338  // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "35:"  // Width 4: Output done
-      "subs x26, x26, #0x4\n"
-      "sub %x[N], %x[N], x27, LSL #2\n"
+      "subs x27, x27, #0x4\n"
+      "sub %x[N], %x[N], x28, LSL #2\n"
       "bgt 4b\n"
       "36:"  // Exit
       ".inst 0xd503467f  // SMSTOP\n"
       "ptrue p1.b\n"
       : [N] "+&r" (N)
       : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp
index 4c0ae2c6bd..9224868e6a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -63,487 +63,487 @@ void sme2_gemv_fp32_mla_16VL (
     __asm__ __volatile__(
       "ptrue p1.b\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
-      "cntw x27, ALL, MUL #4\n"
-      "add x26, %x[N], x27\n"
-      "sub x26, x26, #0x1\n"
-      "udiv x26, x26, x27\n"
-      "add x21, x26, #0x3\n"
-      "and x21, x21, #0xfffffffffffffffc\n"
-      "mul x21, x21, x27\n"
-      "mul x21, x21, %x[K]\n"
+      "cntw x28, ALL, MUL #4\n"
+      "add x27, %x[N], x28\n"
+      "sub x27, x27, #0x1\n"
+      "udiv x27, x27, x28\n"
+      "add x22, x27, #0x3\n"
+      "and x22, x22, #0xfffffffffffffffc\n"
+      "mul x22, x22, x28\n"
+      "mul x22, x22, %x[K]\n"
       "mov x9, #0x0\n"
-      "mov x25, %x[B_ptr]\n"
-      "mov x24, %x[output_ptr]\n"
+      "mov x26, %x[B_ptr]\n"
+      "mov x25, %x[output_ptr]\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
-      "lsl x21, x21, #0x2\n"
-      "mov x20, #0x1\n"
+      "lsl x22, x22, #0x2\n"
+      "mov x21, #0x1\n"
       "1:"  // RHS size check loop
-      "cmp x21, #0x200000\n"
+      "cmp x22, #0x200000\n"
       "blt 2f\n"
-      "tbnz x21, #0, 3f\n"
-      "lsr x21, x21, #0x1\n"
-      "lsl x20, x20, #0x1\n"
+      "tbnz x22, #0, 3f\n"
+      "lsr x22, x22, #0x1\n"
+      "lsl x21, x21, #0x1\n"
       "b 1b\n"
       "2:"  // RHS do prefetch
-      "lsl x19, x21, #0x26\n"
-      "sub x20, x20, #0x1\n"
-      "lsl x20, x20, #0x16\n"
-      "orr x21, x21, x19\n"
-      "orr x21, x21, x20\n"
-      ".inst 0xf8b54b3a  // rprfm pldonce, x21, [x25]\n"
+      "lsl x20, x22, #0x26\n"
+      "sub x21, x21, #0x1\n"
+      "lsl x21, x21, #0x16\n"
+      "orr x22, x22, x20\n"
+      "orr x22, x22, x21\n"
+      ".inst 0xf8b64b5a  // rprfm pldonce, x22, [x26]\n"
       "3:"  // RHS prefetch exit
-      "mov x23, %x[bias]\n"
+      "mov x24, %x[bias]\n"
       "4:"  // Column loop
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "bge 28f\n"
-      "cmp x26, #0x2\n"
+      "cmp x27, #0x2\n"
       "bgt 20f\n"
       "beq 12f\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "mov x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 5f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "mov x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 5f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
       "b 6f\n"
       "5:"  // Width 1: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "6:"  // Width 1: setup done
-      "cmp x20, #0x4\n"
+      "cmp x21, #0x4\n"
       "ble 8f\n"
       "7:"  // Width 1: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x4\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x4\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
-      "cmp x20, #0x4\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      "cmp x21, #0x4\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 7b\n"
       "8:"  // Width 1: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x1\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x1\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "9:"  // Width 1: Multiply loop: multiply skip
       "tbz %x[flags], #1, 10f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "b 11f\n"
       "10:"  // Width 1: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "11:"  // Width 1: Output done
       "b 36f\n"
       "12:"  // Width 2
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "sub x19, %x[N], x27\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 13f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "sub x20, %x[N], x28\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 13f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
       "b 14f\n"
       "13:"  // Width 2: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "14:"  // Width 2: setup done
-      "cmp x20, #0x4\n"
+      "cmp x21, #0x4\n"
       "ble 16f\n"
       "15:"  // Width 2: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x4\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x4\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      "cmp x20, #0x4\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x4\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 15b\n"
       "16:"  // Width 2: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x1\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x1\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "17:"  // Width 2: Multiply loop: multiply skip
       "tbz %x[flags], #1, 18f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
-      ".inst 0xa061c314  // st1w { z20.s-z23.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c334  // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "b 19f\n"
       "18:"  // Width 2: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c314  // st1w { z20.s-z23.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c334  // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "19:"  // Width 2: Output done
       "b 36f\n"
       "20:"  // Width 3
-      "mov x19, #0x2\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 21f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x2\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "msub x20, x28, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 21f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
-      ".inst 0xa042c6e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c704  // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042c82  // mova za.d[x9, #2], { z4.d-z7.d }\n"
       "b 22f\n"
       "21:"  // Width 3: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "22:"  // Width 3: setup done
-      "cmp x20, #0x4\n"
+      "cmp x21, #0x4\n"
       "ble 24f\n"
       "23:"  // Width 3: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x4\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x4\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      "cmp x20, #0x4\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x4\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      ".inst 0xa042c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa282  // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      ".inst 0xa042c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa702  // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      ".inst 0xa042c73d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c75d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aab82  // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      ".inst 0xa042c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aae02  // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 23b\n"
       "24:"  // Width 3: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x1\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x1\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      ".inst 0xa042c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa282  // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      ".inst 0xa042c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa702  // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      ".inst 0xa042c73d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c75d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aab82  // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      ".inst 0xa042c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aae02  // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "25:"  // Width 3: Multiply loop: multiply skip
       "tbz %x[flags], #1, 26f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1a6c810  // fclamp { z16.s-z19.s }, z0.s, z6.s\n"
-      ".inst 0xa062c310  // st1w { z16.s-z19.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c330  // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "b 27f\n"
       "26:"  // Width 3: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c310  // st1w { z16.s-z19.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c330  // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "27:"  // Width 3: Output done
       "b 36f\n"
       "28:"  // Width 4
-      "mov x19, #0x3\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 29f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x3\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "msub x20, x28, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 29f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c708  // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042d01  // mova za.d[x9, #1], { z8.d-z11.d }\n"
-      ".inst 0xa042c6e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c704  // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042c82  // mova za.d[x9, #2], { z4.d-z7.d }\n"
-      ".inst 0xa043c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      ".inst 0xa043c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n"
       ".inst 0xc0042e03  // mova za.d[x9, #3], { z16.d-z19.d }\n"
-      "addvl x23, x23, #16\n"
+      "addvl x24, x24, #16\n"
       "b 30f\n"
       "29:"  // Width 4: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "30:"  // Width 4: setup done
-      "cmp x20, #0x4\n"
+      "cmp x21, #0x4\n"
       "ble 32f\n"
       "31:"  // Width 4: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "sub x20, x20, #0x4\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "sub x21, x21, #0x4\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      "cmp x20, #0x4\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      "cmp x21, #0x4\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      ".inst 0xa042c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa282  // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aa203  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      ".inst 0xa042c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa702  // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aa603  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      ".inst 0xa042c73d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c75d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aab82  // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n"
-      ".inst 0xa043c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aaa83  // fmla za.s[x9, 3], { z20.s-z23.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      ".inst 0xa042c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aae02  // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aae03  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "bgt 31b\n"
       "32:"  // Width 4: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x20\n"
-      "ld1rqw { z10.s }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x1\n"
-      ".inst 0xa040c721  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.s, XZR, x21\n"
+      "ld1rqw { z10.s }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x1\n"
+      ".inst 0xa040c741  // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc15aa000  // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n"
-      ".inst 0xa041c725  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c745  // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa081  // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n"
-      ".inst 0xa042c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa282  // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aa203  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa700  // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aa601  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n"
-      ".inst 0xa042c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aa702  // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aa603  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040c72d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
+      ".inst 0xa040c74d  // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
       ".inst 0xc15aa980  // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n"
-      ".inst 0xa041c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaa01  // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n"
-      ".inst 0xa042c73d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c75d  // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aab82  // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n"
-      ".inst 0xa043c735  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c755  // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aaa83  // fmla za.s[x9, 3], { z20.s-z23.s }, z10.s[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n"
       ".inst 0xc15aae00  // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa041c739  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041c759  // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc15aaf01  // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n"
-      ".inst 0xa042c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc15aae02  // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n"
-      ".inst 0xa043c731  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043c751  // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc15aae03  // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "33:"  // Width 4: Multiply loop: multiply skip
       "tbz %x[flags], #1, 34f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z0.s }, p1/Z, [x20]\n"
+      "ld1rw { z0.s }, p1/Z, [x21]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      "ld1rw { z6.s }, p1/Z, [x19]\n"
+      "ld1rw { z6.s }, p1/Z, [x20]\n"
       ".inst 0xc1a6c808  // fclamp { z8.s-z11.s }, z0.s, z6.s\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1a6c814  // fclamp { z20.s-z23.s }, z0.s, z6.s\n"
       ".inst 0xc0062c78  // mova { z24.d-z27.d }, za.d[x9, #3]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1a6c810  // fclamp { z16.s-z19.s }, z0.s, z6.s\n"
-      ".inst 0xa062c710  // st1w { z16.s-z19.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c730  // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc1a6c818  // fclamp { z24.s-z27.s }, z0.s, z6.s\n"
-      ".inst 0xa063c318  // st1w { z24.s-z27.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c338  // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "b 35f\n"
       "34:"  // Width 4: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c34  // mova { z20.d-z23.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c714  // st1w { z20.s-z23.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c734  // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c50  // mova { z16.d-z19.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c710  // st1w { z16.s-z19.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c730  // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc0062c78  // mova { z24.d-z27.d }, za.d[x9, #3]\n"
-      ".inst 0xa063c318  // st1w { z24.s-z27.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c338  // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "35:"  // Width 4: Output done
-      "subs x26, x26, #0x4\n"
-      "sub %x[N], %x[N], x27, LSL #2\n"
+      "subs x27, x27, #0x4\n"
+      "sub %x[N], %x[N], x28, LSL #2\n"
       "bgt 4b\n"
       "36:"  // Exit
       ".inst 0xd503467f  // SMSTOP\n"
       "ptrue p1.b\n"
       : [N] "+&r" (N)
       : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp
index 8b8bcb6bc7..0a394b6413 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -64,544 +64,544 @@ void sme2_gemv_fp32bf16fp32_dot_16VL (
     __asm__ __volatile__(
       "ptrue p2.b\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
-      "cntw x28, ALL, MUL #4\n"
-      "add x27, %x[N], x28\n"
-      "sub x27, x27, #0x1\n"
-      "udiv x27, x27, x28\n"
-      "add x21, x27, #0x3\n"
-      "and x21, x21, #0xfffffffffffffffc\n"
-      "mul x21, x21, x28\n"
-      "mul x21, x21, %x[K]\n"
+      "cntw x10, ALL, MUL #4\n"
+      "add x28, %x[N], x10\n"
+      "sub x28, x28, #0x1\n"
+      "udiv x28, x28, x10\n"
+      "add x22, x28, #0x3\n"
+      "and x22, x22, #0xfffffffffffffffc\n"
+      "mul x22, x22, x10\n"
+      "mul x22, x22, %x[K]\n"
       "mov x9, #0x0\n"
-      "mov x26, #0x4\n"
-      "mov x25, %x[B_ptr]\n"
-      "mov x24, %x[output_ptr]\n"
+      "mov x27, #0x4\n"
+      "mov x26, %x[B_ptr]\n"
+      "mov x25, %x[output_ptr]\n"
       "ptrue p2.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
-      "lsl x21, x21, #0x1\n"
-      "mov x20, #0x1\n"
+      "lsl x22, x22, #0x1\n"
+      "mov x21, #0x1\n"
       "1:"  // RHS size check loop
-      "cmp x21, #0x200000\n"
+      "cmp x22, #0x200000\n"
       "blt 2f\n"
-      "tbnz x21, #0, 3f\n"
-      "lsr x21, x21, #0x1\n"
-      "lsl x20, x20, #0x1\n"
+      "tbnz x22, #0, 3f\n"
+      "lsr x22, x22, #0x1\n"
+      "lsl x21, x21, #0x1\n"
       "b 1b\n"
       "2:"  // RHS do prefetch
-      "lsl x19, x21, #0x26\n"
-      "sub x20, x20, #0x1\n"
-      "lsl x20, x20, #0x16\n"
-      "orr x21, x21, x19\n"
-      "orr x21, x21, x20\n"
-      ".inst 0xf8b54b3a  // rprfm pldonce, x21, [x25]\n"
+      "lsl x20, x22, #0x26\n"
+      "sub x21, x21, #0x1\n"
+      "lsl x21, x21, #0x16\n"
+      "orr x22, x22, x20\n"
+      "orr x22, x22, x21\n"
+      ".inst 0xf8b64b5a  // rprfm pldonce, x22, [x26]\n"
       "3:"  // RHS prefetch exit
-      "mov x23, %x[bias]\n"
+      "mov x24, %x[bias]\n"
       "4:"  // Column loop
-      "cmp x27, #0x4\n"
+      "cmp x28, #0x4\n"
       "bge 28f\n"
-      "cmp x27, #0x2\n"
+      "cmp x28, #0x2\n"
       "bgt 20f\n"
       "beq 12f\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "mov x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 5f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "mov x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 5f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
       "b 6f\n"
       "5:"  // Width 1: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "6:"  // Width 1: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 8f\n"
       "7:"  // Width 1: Multiply loop: Main loop head
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "sub x20, x20, #0x8\n"
+      "sub x21, x21, #0x8\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "addvl x25, x25, #16\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "addvl x25, x25, #16\n"
-      "cmp x20, #0x8\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "addvl x26, x26, #16\n"
+      "cmp x21, #0x8\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "addvl x25, x25, #16\n"
-      "add x22, x22, #0x20\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "addvl x26, x26, #16\n"
+      "add x23, x23, #0x20\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
-      "addvl x25, x25, #16\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
       "bgt 7b\n"
       "8:"  // Width 1: Multiply loop: Single iteration only
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "subs x20, x20, #0x2\n"
+      "subs x21, x21, #0x2\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x20\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x20\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 9f\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "9:"  // Width 1: Multiply loop: multiply skip
       "tbz %x[flags], #1, 10f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z29.s }, p2/Z, [x20]\n"
-      "ld1rw { z18.s }, p2/Z, [x19]\n"
+      "ld1rw { z29.s }, p2/Z, [x21]\n"
+      "ld1rw { z18.s }, p2/Z, [x20]\n"
       ".inst 0xc1b2cba8  // fclamp { z8.s-z11.s }, z29.s, z18.s\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "b 11f\n"
       "10:"  // Width 1: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c308  // st1w { z8.s-z11.s }, p8, [x24]\n"
-      "addvl x24, x24, #4\n"
+      ".inst 0xa060c328  // st1w { z8.s-z11.s }, p8, [x25]\n"
+      "addvl x25, x25, #4\n"
       "11:"  // Width 1: Output done
       "b 36f\n"
       "12:"  // Width 2
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "sub x19, %x[N], x28\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 13f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "sub x20, %x[N], x10\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 13f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e01  // mova za.d[x9, #1], { z16.d-z19.d }\n"
       "b 14f\n"
       "13:"  // Width 2: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "14:"  // Width 2: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 16f\n"
       "15:"  // Width 2: Multiply loop: Main loop head
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "sub x20, x20, #0x8\n"
+      "sub x21, x21, #0x8\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "cmp x20, #0x8\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "cmp x21, #0x8\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
-      "add x22, x22, #0x20\n"
+      "addvl x26, x26, #16\n"
+      "add x23, x23, #0x20\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
       "bgt 15b\n"
       "16:"  // Width 2: Multiply loop: Single iteration only
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "subs x20, x20, #0x2\n"
+      "subs x21, x21, #0x2\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x20\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x20\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
       "ble 17f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 17f\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "17:"  // Width 2: Multiply loop: multiply skip
       "tbz %x[flags], #1, 18f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z29.s }, p2/Z, [x20]\n"
+      "ld1rw { z29.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      "ld1rw { z18.s }, p2/Z, [x19]\n"
+      "ld1rw { z18.s }, p2/Z, [x20]\n"
       ".inst 0xc1b2cba8  // fclamp { z8.s-z11.s }, z29.s, z18.s\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1b2cbac  // fclamp { z12.s-z15.s }, z29.s, z18.s\n"
-      ".inst 0xa061c30c  // st1w { z12.s-z15.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c32c  // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "b 19f\n"
       "18:"  // Width 2: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c30c  // st1w { z12.s-z15.s }, p8, [x24, #0x4, MUL VL]\n"
-      "addvl x24, x24, #8\n"
+      ".inst 0xa061c32c  // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n"
+      "addvl x25, x25, #8\n"
       "19:"  // Width 2: Output done
       "b 36f\n"
       "20:"  // Width 3
-      "mov x19, #0x2\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "msub x19, x28, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 21f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x2\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "msub x20, x10, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 21f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e01  // mova za.d[x9, #1], { z16.d-z19.d }\n"
-      ".inst 0xa042c6fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c71c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042f82  // mova za.d[x9, #2], { z28.d-z31.d }\n"
       "b 22f\n"
       "21:"  // Width 3: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "22:"  // Width 3: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 24f\n"
       "23:"  // Width 3: Multiply loop: Main loop head
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "sub x20, x20, #0x8\n"
+      "sub x21, x21, #0x8\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "cmp x20, #0x8\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "cmp x21, #0x8\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      "add x22, x22, #0x20\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      "add x23, x23, #0x20\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b21a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa042a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b59a  // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150ba1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150be1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n"
       "bgt 23b\n"
       "24:"  // Width 3: Multiply loop: Single iteration only
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "subs x20, x20, #0x2\n"
+      "subs x21, x21, #0x2\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x20\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x20\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b21a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n"
       "ble 25f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      ".inst 0xa042a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b59a  // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150ba1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 25f\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150be1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "25:"  // Width 3: Multiply loop: multiply skip
       "tbz %x[flags], #1, 26f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z29.s }, p2/Z, [x20]\n"
+      "ld1rw { z29.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      "ld1rw { z18.s }, p2/Z, [x19]\n"
+      "ld1rw { z18.s }, p2/Z, [x20]\n"
       ".inst 0xc1b2cba8  // fclamp { z8.s-z11.s }, z29.s, z18.s\n"
       ".inst 0xc0062c44  // mova { z4.d-z7.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1b2cbac  // fclamp { z12.s-z15.s }, z29.s, z18.s\n"
-      ".inst 0xa061c70c  // st1w { z12.s-z15.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c72c  // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1b2cba4  // fclamp { z4.s-z7.s }, z29.s, z18.s\n"
-      ".inst 0xa062c304  // st1w { z4.s-z7.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c324  // st1w { z4.s-z7.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "b 27f\n"
       "26:"  // Width 3: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c70c  // st1w { z12.s-z15.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c72c  // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c44  // mova { z4.d-z7.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c304  // st1w { z4.s-z7.s }, p8, [x24, #0x8, MUL VL]\n"
-      "addvl x24, x24, #12\n"
+      ".inst 0xa062c324  // st1w { z4.s-z7.s }, p8, [x25, #0x8, MUL VL]\n"
+      "addvl x25, x25, #12\n"
       "27:"  // Width 3: Output done
       "b 36f\n"
       "28:"  // Width 4
-      "mov x19, #0x3\n"
-      "mov x22, %x[A_ptr]\n"
-      "lsl x21, %x[K], #0x2\n"
-      "msub x19, x28, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      ".inst 0x25b367f0  // whilelt p8.s, XZR, x19, VLx4\n"
-      "cbz x23, 29f\n"
-      ".inst 0xa040c6e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x23]\n"
+      "mov x20, #0x3\n"
+      "mov x23, %x[A_ptr]\n"
+      "lsl x22, %x[K], #0x2\n"
+      "msub x20, x10, x20, %x[N]\n"
+      "mov x21, %x[K]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      ".inst 0x25b467f0  // whilelt p8.s, XZR, x20, VLx4\n"
+      "cbz x24, 29f\n"
+      ".inst 0xa040c700  // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n"
       ".inst 0xc0042c00  // mova za.d[x9, #0], { z0.d-z3.d }\n"
-      ".inst 0xa041c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e01  // mova za.d[x9, #1], { z16.d-z19.d }\n"
-      ".inst 0xa042c6fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c71c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042f82  // mova za.d[x9, #2], { z28.d-z31.d }\n"
-      ".inst 0xa043c6f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      ".inst 0xa043c710  // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n"
       ".inst 0xc0042e03  // mova za.d[x9, #3], { z16.d-z19.d }\n"
-      "addvl x23, x23, #16\n"
+      "addvl x24, x24, #16\n"
       "b 30f\n"
       "29:"  // Width 4: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "30:"  // Width 4: setup done
-      "cmp x20, #0x8\n"
+      "cmp x21, #0x8\n"
       "ble 32f\n"
       "31:"  // Width 4: Multiply loop: Main loop head
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "sub x20, x20, #0x8\n"
+      "sub x21, x21, #0x8\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "cmp x20, #0x8\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "cmp x21, #0x8\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      "add x22, x22, #0x20\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      "add x23, x23, #0x20\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150b21a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b39b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[0]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa042a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150b59a  // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b79b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[1]\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      ".inst 0xa043a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150ba1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b99b  // bfdot za.s[x9, 3], { z12.h-z15.h }, z0.h[2]\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150be1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150bf9b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[3]\n"
       "bgt 31b\n"
       "32:"  // Width 4: Multiply loop: Single iteration only
-      "whilelt p1.s, XZR, x20\n"
-      "whilelt p0.s, x26, x20\n"
-      "ld1rqw { z0.s }, p1/Z, [x22]\n"
+      "whilelt p1.s, XZR, x21\n"
+      "whilelt p0.s, x27, x21\n"
+      "ld1rqw { z0.s }, p1/Z, [x23]\n"
       ".inst 0x658aa800  // bfcvt z0.h, p2/M, z0.s\n"
-      "ld1rqw { z11.s }, p0/Z, [x22, #16]\n"
+      "ld1rqw { z11.s }, p0/Z, [x23, #16]\n"
       ".inst 0x658aa96b  // bfcvt z11.h, p2/M, z11.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "subs x20, x20, #0x2\n"
+      "subs x21, x21, #0x2\n"
       "uzp1 z11.h, z11.h, z11.h\n"
       "trn1 z0.d, z0.d, z11.d\n"
-      ".inst 0xa040a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25]\n"
-      "add x22, x22, #0x20\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa040a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n"
+      "add x23, x23, #0x20\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b098  // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b119  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150b21a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       ".inst 0xc150b39b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[0]\n"
       "ble 33f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150b618  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n"
-      ".inst 0xa041a725  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a745  // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150b499  // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n"
-      ".inst 0xa042a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150b59a  // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150b79b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25]\n"
-      "subs x20, x20, #0x2\n"
+      ".inst 0xa040a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n"
+      "subs x21, x21, #0x2\n"
       ".inst 0xc150ba18  // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa041a735  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a755  // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150ba99  // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150ba1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n"
-      ".inst 0xa043a72d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a74d  // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150b99b  // bfdot za.s[x9, 3], { z12.h-z15.h }, z0.h[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 33f\n"
-      ".inst 0xa040a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25]\n"
+      ".inst 0xa040a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n"
       ".inst 0xc150bf98  // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n"
-      ".inst 0xa041a729  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa041a749  // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc150bd19  // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n"
-      ".inst 0xa042a731  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042a751  // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc150be1a  // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n"
-      ".inst 0xa043a73d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043a75d  // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc150bf9b  // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "33:"  // Width 4: Multiply loop: multiply skip
       "tbz %x[flags], #1, 34f\n"
-      "add x20, %x[args_ptr], %[offset_min]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
+      "add x21, %x[args_ptr], %[offset_min]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      "ld1rw { z29.s }, p2/Z, [x20]\n"
+      "ld1rw { z29.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      "ld1rw { z18.s }, p2/Z, [x19]\n"
+      "ld1rw { z18.s }, p2/Z, [x20]\n"
       ".inst 0xc1b2cba8  // fclamp { z8.s-z11.s }, z29.s, z18.s\n"
       ".inst 0xc0062c44  // mova { z4.d-z7.d }, za.d[x9, #2]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc1b2cbac  // fclamp { z12.s-z15.s }, z29.s, z18.s\n"
       ".inst 0xc0062c60  // mova { z0.d-z3.d }, za.d[x9, #3]\n"
-      ".inst 0xa061c70c  // st1w { z12.s-z15.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c72c  // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc1b2cba4  // fclamp { z4.s-z7.s }, z29.s, z18.s\n"
-      ".inst 0xa062c704  // st1w { z4.s-z7.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c724  // st1w { z4.s-z7.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc1b2cba0  // fclamp { z0.s-z3.s }, z29.s, z18.s\n"
-      ".inst 0xa063c300  // st1w { z0.s-z3.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c320  // st1w { z0.s-z3.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "b 35f\n"
       "34:"  // Width 4: No activation
       ".inst 0xc0062c08  // mova { z8.d-z11.d }, za.d[x9, #0]\n"
-      ".inst 0xa060c708  // st1w { z8.s-z11.s }, pn9.b, [x24]\n"
+      ".inst 0xa060c728  // st1w { z8.s-z11.s }, pn9.b, [x25]\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
-      ".inst 0xa061c70c  // st1w { z12.s-z15.s }, pn9.b, [x24, #0x4, MUL VL]\n"
+      ".inst 0xa061c72c  // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n"
       ".inst 0xc0062c44  // mova { z4.d-z7.d }, za.d[x9, #2]\n"
-      ".inst 0xa062c704  // st1w { z4.s-z7.s }, pn9.b, [x24, #0x8, MUL VL]\n"
+      ".inst 0xa062c724  // st1w { z4.s-z7.s }, pn9.b, [x25, #0x8, MUL VL]\n"
       ".inst 0xc0062c60  // mova { z0.d-z3.d }, za.d[x9, #3]\n"
-      ".inst 0xa063c300  // st1w { z0.s-z3.s }, p8, [x24, #0xc, MUL VL]\n"
-      "addvl x24, x24, #16\n"
+      ".inst 0xa063c320  // st1w { z0.s-z3.s }, p8, [x25, #0xc, MUL VL]\n"
+      "addvl x25, x25, #16\n"
       "35:"  // Width 4: Output done
-      "subs x27, x27, #0x4\n"
-      "sub %x[N], %x[N], x28, LSL #2\n"
+      "subs x28, x28, #0x4\n"
+      "sub %x[N], %x[N], x10, LSL #2\n"
       "bgt 4b\n"
       "36:"  // Exit
       ".inst 0xd503467f  // SMSTOP\n"
       "ptrue p2.b\n"
       : [N] "+&r" (N)
       : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp
index 348c709119..26dc0b9dd2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -54,257 +54,257 @@ void sme2_gemv_s8qa_dot_16VL (
     __asm__ __volatile__(
       "ptrue p2.b\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
-      "cntw x27, ALL, MUL #4\n"
-      "add x26, %x[N], x27\n"
-      "sub x26, x26, #0x1\n"
-      "udiv x26, x26, x27\n"
-      "add x21, x26, #0x3\n"
-      "and x21, x21, #0xfffffffffffffffc\n"
-      "mul x21, x21, x27\n"
+      "cntw x28, ALL, MUL #4\n"
+      "add x27, %x[N], x28\n"
+      "sub x27, x27, #0x1\n"
+      "udiv x27, x27, x28\n"
+      "add x22, x27, #0x3\n"
+      "and x22, x22, #0xfffffffffffffffc\n"
+      "mul x22, x22, x28\n"
       "mov x9, #0x0\n"
-      "mov x25, %x[B_ptr]\n"
-      "mov x24, %x[output_ptr]\n"
+      "mov x26, %x[B_ptr]\n"
+      "mov x25, %x[output_ptr]\n"
       "ptrue p2.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
-      "mul x21, x21, %x[K]\n"
-      "mov x20, #0x1\n"
+      "mul x22, x22, %x[K]\n"
+      "mov x21, #0x1\n"
       "1:"  // RHS size check loop
-      "cmp x21, #0x200000\n"
+      "cmp x22, #0x200000\n"
       "blt 2f\n"
-      "tbnz x21, #0, 3f\n"
-      "lsr x21, x21, #0x1\n"
-      "lsl x20, x20, #0x1\n"
+      "tbnz x22, #0, 3f\n"
+      "lsr x22, x22, #0x1\n"
+      "lsl x21, x21, #0x1\n"
       "b 1b\n"
       "2:"  // RHS do prefetch
-      "lsl x19, x21, #0x26\n"
-      "sub x20, x20, #0x1\n"
-      "lsl x20, x20, #0x16\n"
-      "orr x21, x21, x19\n"
-      "orr x21, x21, x20\n"
-      ".inst 0xf8b54b3a  // rprfm pldonce, x21, [x25]\n"
+      "lsl x20, x22, #0x26\n"
+      "sub x21, x21, #0x1\n"
+      "lsl x21, x21, #0x16\n"
+      "orr x22, x22, x20\n"
+      "orr x22, x22, x21\n"
+      ".inst 0xf8b64b5a  // rprfm pldonce, x22, [x26]\n"
       "3:"  // RHS prefetch exit
-      "mov x23, %x[col_bias]\n"
+      "mov x24, %x[col_bias]\n"
       "mov z26.s, #0x0\n"
       "mov z24.b, #0x1\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "4:"  // Column loop
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "bge 34f\n"
-      "cmp x26, #0x2\n"
+      "cmp x27, #0x2\n"
       "bgt 24f\n"
       "beq 14f\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "mov x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "mov x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 5f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 5f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
       "b 6f\n"
       "5:"  // Width 1: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "6:"  // Width 1: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 9f\n"
       "7:"  // Width 1: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 8f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "8:"  // Width 1: Multiply loop: unique 1: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 7b\n"
       "9:"  // Width 1: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "10:"  // Width 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "11:"  // Width 1: Multiply loop: unique 2: skip row sum
       "tbnz %x[flags], #31, 12f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "saddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "12:"  // Width 1: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1b0cebc  // sclamp { z28.s-z31.s }, z21.s, z16.s\n"
       "uzp1 z28.h, z28.h, z29.h\n"
       "uzp1 z29.h, z30.h, z31.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p1, [x24]\n"
-      "addvl x24, x24, #1\n"
+      "st1b { z28.b }, p1, [x25]\n"
+      "addvl x25, x25, #1\n"
       "13:"  // Width 1: Output done
       "b 44f\n"
       "14:"  // Width 2
-      "mov x22, %x[A_ptr]\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "sub x20, %x[N], x28\n"
       "mov x21, %x[K]\n"
-      "sub x19, %x[N], x27\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 15f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 15f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
       "b 16f\n"
       "15:"  // Width 2: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "16:"  // Width 2: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 19f\n"
       "17:"  // Width 2: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 18f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "18:"  // Width 2: Multiply loop: unique 3: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 17b\n"
       "19:"  // Width 2: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "20:"  // Width 2: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 21f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "21:"  // Width 2: Multiply loop: unique 4: skip row sum
       "tbnz %x[flags], #31, 22f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "saddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "22:"  // Width 2: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
       ".inst 0xc1a5ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
@@ -315,137 +315,137 @@ void sme2_gemv_s8qa_dot_16VL (
       "uzp1 z12.h, z12.h, z13.h\n"
       "uzp1 z13.h, z14.h, z15.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x24, #1, MUL VL]\n"
-      "addvl x24, x24, #2\n"
+      "st1b { z12.b }, p1, [x25, #1, MUL VL]\n"
+      "addvl x25, x25, #2\n"
       "23:"  // Width 2: Output done
       "b 44f\n"
       "24:"  // Width 3
-      "mov x19, #0x2\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x20, #0x2\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "msub x20, x28, x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 25f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 25f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
-      ".inst 0xa042c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042e82  // mova za.d[x9, #2], { z20.d-z23.d }\n"
       "b 26f\n"
       "25:"  // Width 3: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "26:"  // Width 3: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 29f\n"
       "27:"  // Width 3: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b622  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be22  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 28f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "28:"  // Width 3: Multiply loop: unique 5: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 27b\n"
       "29:"  // Width 3: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b622  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be22  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "30:"  // Width 3: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 31f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "31:"  // Width 3: Multiply loop: unique 6: skip row sum
       "tbnz %x[flags], #31, 32f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "saddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "32:"  // Width 3: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
@@ -453,7 +453,7 @@ void sme2_gemv_s8qa_dot_16VL (
       ".inst 0xc0062c40  // mova { z0.d-z3.d }, za.d[x9, #2]\n"
       ".inst 0xc1a5ac00  // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a4aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
@@ -469,158 +469,158 @@ void sme2_gemv_s8qa_dot_16VL (
       "uzp1 z0.h, z0.h, z1.h\n"
       "uzp1 z1.h, z2.h, z3.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p2, [x24, #1, MUL VL]\n"
+      "st1b { z12.b }, p2, [x25, #1, MUL VL]\n"
       "uzp1 z0.b, z0.b, z1.b\n"
-      "st1b { z0.b }, p1, [x24, #2, MUL VL]\n"
-      "addvl x24, x24, #3\n"
+      "st1b { z0.b }, p1, [x25, #2, MUL VL]\n"
+      "addvl x25, x25, #3\n"
       "33:"  // Width 3: Output done
       "b 44f\n"
       "34:"  // Width 4
-      "mov x19, #0x3\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x20, #0x3\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "msub x20, x28, x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 35f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 35f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
-      ".inst 0xa042c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042e82  // mova za.d[x9, #2], { z20.d-z23.d }\n"
-      ".inst 0xa043c2f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x23, #0xc, MUL VL]\n"
+      ".inst 0xa043c310  // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0xc, MUL VL]\n"
       ".inst 0xc0042e03  // mova za.d[x9, #3], { z16.d-z19.d }\n"
-      "addvl x23, x23, #16\n"
+      "addvl x24, x24, #16\n"
       "b 36f\n"
       "35:"  // Width 4: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "36:"  // Width 4: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 39f\n"
       "37:"  // Width 4: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b1a3  // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b622  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b5a3  // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      ".inst 0xa043833d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043835d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153bba3  // sdot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be22  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      ".inst 0xa0438331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa0438351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153be23  // sdot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 38f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "38:"  // Width 4: Multiply loop: unique 7: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 37b\n"
       "39:"  // Width 4: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b220  // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b1a3  // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6a0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b521  // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b622  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b5a3  // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153baa0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8a1  // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9a2  // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      ".inst 0xa043833d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043835d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153bba3  // sdot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bea0  // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153bea1  // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be22  // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      ".inst 0xa0438331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa0438351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153be23  // sdot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "40:"  // Width 4: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 41f\n"
       "sdot z26.s, z3.b, z24.b\n"
       "41:"  // Width 4: Multiply loop: unique 8: skip row sum
       "tbnz %x[flags], #31, 42f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "saddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "42:"  // Width 4: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
@@ -630,7 +630,7 @@ void sme2_gemv_s8qa_dot_16VL (
       ".inst 0xc0062c68  // mova { z8.d-z11.d }, za.d[x9, #3]\n"
       ".inst 0xc1a5ac08  // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a4aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n"
       ".inst 0xc1a4aa28  // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n"
@@ -651,24 +651,24 @@ void sme2_gemv_s8qa_dot_16VL (
       "uzp1 z8.h, z8.h, z9.h\n"
       "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p2, [x24, #1, MUL VL]\n"
+      "st1b { z12.b }, p2, [x25, #1, MUL VL]\n"
       "uzp1 z0.b, z0.b, z1.b\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z0.b }, p2, [x24, #2, MUL VL]\n"
-      "st1b { z8.b }, p1, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
+      "st1b { z0.b }, p2, [x25, #2, MUL VL]\n"
+      "st1b { z8.b }, p1, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
       "43:"  // Width 4: Output done
-      "subs x26, x26, #0x4\n"
-      "sub %x[N], %x[N], x27, LSL #2\n"
+      "subs x27, x27, #0x4\n"
+      "sub %x[N], %x[N], x28, LSL #2\n"
       "bgt 4b\n"
       "44:"  // Exit
       ".inst 0xd503467f  // SMSTOP\n"
       "ptrue p2.b\n"
       : [N] "+&r" (N), [flags] "+&r" (flags)
       : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [output_ptr] "r" (output_ptr), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp
index 9822f637fb..dfdc4ea289 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -54,257 +54,257 @@ void sme2_gemv_u8qa_dot_16VL (
     __asm__ __volatile__(
       "ptrue p2.b\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
-      "cntw x27, ALL, MUL #4\n"
-      "add x26, %x[N], x27\n"
-      "sub x26, x26, #0x1\n"
-      "udiv x26, x26, x27\n"
-      "add x21, x26, #0x3\n"
-      "and x21, x21, #0xfffffffffffffffc\n"
-      "mul x21, x21, x27\n"
+      "cntw x28, ALL, MUL #4\n"
+      "add x27, %x[N], x28\n"
+      "sub x27, x27, #0x1\n"
+      "udiv x27, x27, x28\n"
+      "add x22, x27, #0x3\n"
+      "and x22, x22, #0xfffffffffffffffc\n"
+      "mul x22, x22, x28\n"
       "mov x9, #0x0\n"
-      "mov x25, %x[B_ptr]\n"
-      "mov x24, %x[output_ptr]\n"
+      "mov x26, %x[B_ptr]\n"
+      "mov x25, %x[output_ptr]\n"
       "ptrue p2.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
-      "mul x21, x21, %x[K]\n"
-      "mov x20, #0x1\n"
+      "mul x22, x22, %x[K]\n"
+      "mov x21, #0x1\n"
       "1:"  // RHS size check loop
-      "cmp x21, #0x200000\n"
+      "cmp x22, #0x200000\n"
       "blt 2f\n"
-      "tbnz x21, #0, 3f\n"
-      "lsr x21, x21, #0x1\n"
-      "lsl x20, x20, #0x1\n"
+      "tbnz x22, #0, 3f\n"
+      "lsr x22, x22, #0x1\n"
+      "lsl x21, x21, #0x1\n"
       "b 1b\n"
       "2:"  // RHS do prefetch
-      "lsl x19, x21, #0x26\n"
-      "sub x20, x20, #0x1\n"
-      "lsl x20, x20, #0x16\n"
-      "orr x21, x21, x19\n"
-      "orr x21, x21, x20\n"
-      ".inst 0xf8b54b3a  // rprfm pldonce, x21, [x25]\n"
+      "lsl x20, x22, #0x26\n"
+      "sub x21, x21, #0x1\n"
+      "lsl x21, x21, #0x16\n"
+      "orr x22, x22, x20\n"
+      "orr x22, x22, x21\n"
+      ".inst 0xf8b64b5a  // rprfm pldonce, x22, [x26]\n"
       "3:"  // RHS prefetch exit
-      "mov x23, %x[col_bias]\n"
+      "mov x24, %x[col_bias]\n"
       "mov z26.s, #0x0\n"
       "mov z24.b, #0x1\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "4:"  // Column loop
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "bge 34f\n"
-      "cmp x26, #0x2\n"
+      "cmp x27, #0x2\n"
       "bgt 24f\n"
       "beq 14f\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "mov x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "mov x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 5f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 5f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
       "b 6f\n"
       "5:"  // Width 1: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "6:"  // Width 1: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 9f\n"
       "7:"  // Width 1: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 8f\n"
       "udot z26.s, z3.b, z24.b\n"
       "8:"  // Width 1: Multiply loop: unique 1: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 7b\n"
       "9:"  // Width 1: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 10f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "10:"  // Width 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "udot z26.s, z3.b, z24.b\n"
       "11:"  // Width 1: Multiply loop: unique 2: skip row sum
       "tbnz %x[flags], #31, 12f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "uaddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "12:"  // Width 1: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1b0cebc  // sclamp { z28.s-z31.s }, z21.s, z16.s\n"
       "uzp1 z28.h, z28.h, z29.h\n"
       "uzp1 z29.h, z30.h, z31.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p1, [x24]\n"
-      "addvl x24, x24, #1\n"
+      "st1b { z28.b }, p1, [x25]\n"
+      "addvl x25, x25, #1\n"
       "13:"  // Width 1: Output done
       "b 44f\n"
       "14:"  // Width 2
-      "mov x22, %x[A_ptr]\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "sub x20, %x[N], x28\n"
       "mov x21, %x[K]\n"
-      "sub x19, %x[N], x27\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 15f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 15f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
       "b 16f\n"
       "15:"  // Width 2: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "16:"  // Width 2: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 19f\n"
       "17:"  // Width 2: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 18f\n"
       "udot z26.s, z3.b, z24.b\n"
       "18:"  // Width 2: Multiply loop: unique 3: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 17b\n"
       "19:"  // Width 2: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 20f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "20:"  // Width 2: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 21f\n"
       "udot z26.s, z3.b, z24.b\n"
       "21:"  // Width 2: Multiply loop: unique 4: skip row sum
       "tbnz %x[flags], #31, 22f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "uaddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "22:"  // Width 2: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
       ".inst 0xc1a5ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
@@ -315,137 +315,137 @@ void sme2_gemv_u8qa_dot_16VL (
       "uzp1 z12.h, z12.h, z13.h\n"
       "uzp1 z13.h, z14.h, z15.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x24, #1, MUL VL]\n"
-      "addvl x24, x24, #2\n"
+      "st1b { z12.b }, p1, [x25, #1, MUL VL]\n"
+      "addvl x25, x25, #2\n"
       "23:"  // Width 2: Output done
       "b 44f\n"
       "24:"  // Width 3
-      "mov x19, #0x2\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x20, #0x2\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "msub x20, x28, x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 25f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 25f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
-      ".inst 0xa042c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042e82  // mova za.d[x9, #2], { z20.d-z23.d }\n"
       "b 26f\n"
       "25:"  // Width 3: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "26:"  // Width 3: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 29f\n"
       "27:"  // Width 3: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b632  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be32  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 28f\n"
       "udot z26.s, z3.b, z24.b\n"
       "28:"  // Width 3: Multiply loop: unique 5: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 27b\n"
       "29:"  // Width 3: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b632  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 30f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be32  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "30:"  // Width 3: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 31f\n"
       "udot z26.s, z3.b, z24.b\n"
       "31:"  // Width 3: Multiply loop: unique 6: skip row sum
       "tbnz %x[flags], #31, 32f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "uaddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "32:"  // Width 3: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
@@ -453,7 +453,7 @@ void sme2_gemv_u8qa_dot_16VL (
       ".inst 0xc0062c40  // mova { z0.d-z3.d }, za.d[x9, #2]\n"
       ".inst 0xc1a5ac00  // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a4aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
@@ -469,158 +469,158 @@ void sme2_gemv_u8qa_dot_16VL (
       "uzp1 z0.h, z0.h, z1.h\n"
       "uzp1 z1.h, z2.h, z3.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p2, [x24, #1, MUL VL]\n"
+      "st1b { z12.b }, p2, [x25, #1, MUL VL]\n"
       "uzp1 z0.b, z0.b, z1.b\n"
-      "st1b { z0.b }, p1, [x24, #2, MUL VL]\n"
-      "addvl x24, x24, #3\n"
+      "st1b { z0.b }, p1, [x25, #2, MUL VL]\n"
+      "addvl x25, x25, #3\n"
       "33:"  // Width 3: Output done
       "b 44f\n"
       "34:"  // Width 4
-      "mov x19, #0x3\n"
-      "mov x22, %x[A_ptr]\n"
+      "mov x20, #0x3\n"
+      "mov x23, %x[A_ptr]\n"
+      "mov x22, %x[K]\n"
+      "msub x20, x28, x20, %x[N]\n"
       "mov x21, %x[K]\n"
-      "msub x19, x27, x19, %x[N]\n"
-      "mov x20, %x[K]\n"
-      ".inst 0xf8b54ad8  // rprfm pldmany, x21, [x22]\n"
-      "whilelt p1.b, XZR, x19\n"
-      "cbz x23, 35f\n"
-      ".inst 0xa040c2e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x23]\n"
+      ".inst 0xf8b64af8  // rprfm pldmany, x22, [x23]\n"
+      "whilelt p1.b, XZR, x20\n"
+      "cbz x24, 35f\n"
+      ".inst 0xa040c304  // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n"
       ".inst 0xc0042c80  // mova za.d[x9, #0], { z4.d-z7.d }\n"
-      ".inst 0xa041c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa041c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n"
       ".inst 0xc0042e81  // mova za.d[x9, #1], { z20.d-z23.d }\n"
-      ".inst 0xa042c2f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x23, #0x8, MUL VL]\n"
+      ".inst 0xa042c314  // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n"
       ".inst 0xc0042e82  // mova za.d[x9, #2], { z20.d-z23.d }\n"
-      ".inst 0xa043c2f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x23, #0xc, MUL VL]\n"
+      ".inst 0xa043c310  // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0xc, MUL VL]\n"
       ".inst 0xc0042e03  // mova za.d[x9, #3], { z16.d-z19.d }\n"
-      "addvl x23, x23, #16\n"
+      "addvl x24, x24, #16\n"
       "b 36f\n"
       "35:"  // Width 4: no bias
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
       "36:"  // Width 4: setup done
-      "cmp x20, #0x10\n"
+      "cmp x21, #0x10\n"
       "ble 39f\n"
       "37:"  // Width 4: Multiply loop: Main loop head
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b1b3  // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b632  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b5b3  // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      ".inst 0xa043833d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043835d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153bbb3  // udot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      "addvl x26, x26, #16\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be32  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      ".inst 0xa0438331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa0438351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153be33  // udot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "tbnz %x[flags], #31, 38f\n"
       "udot z26.s, z3.b, z24.b\n"
       "38:"  // Width 4: Multiply loop: unique 7: skip row sum
-      "sub x20, x20, #0x10\n"
-      "cmp x20, #0x10\n"
+      "sub x21, x21, #0x10\n"
+      "cmp x21, #0x10\n"
       "bgt 37b\n"
       "39:"  // Width 4: Multiply loop: Single iteration only
-      "whilelt p0.b, XZR, x20\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "subs x20, x20, #0x4\n"
-      ".inst 0xa0408331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25]\n"
-      "add x22, x22, #0x10\n"
+      "whilelt p0.b, XZR, x21\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "subs x21, x21, #0x4\n"
+      ".inst 0xa0408351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0xc153b230  // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b0b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b1b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b1b3  // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153b6b0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n"
-      ".inst 0xa0418329  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418349  // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b531  // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b632  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n"
-      ".inst 0xa043832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153b5b3  // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
-      "subs x20, x20, #0x4\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
+      "subs x21, x21, #0x4\n"
       ".inst 0xc153bab0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n"
-      ".inst 0xa0418325  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418345  // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153b8b1  // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n"
-      ".inst 0xa042832d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa042834d  // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153b9b2  // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n"
-      ".inst 0xa043833d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa043835d  // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153bbb3  // udot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "ble 40f\n"
-      ".inst 0xa0408335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25]\n"
+      ".inst 0xa0408355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n"
       ".inst 0xc153beb0  // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0418335  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x25, #0x4, MUL VL]\n"
+      ".inst 0xa0418355  // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
       ".inst 0xc153beb1  // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n"
-      ".inst 0xa0428331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0x8, MUL VL]\n"
+      ".inst 0xa0428351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
       ".inst 0xc153be32  // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n"
-      ".inst 0xa0438331  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x25, #0xc, MUL VL]\n"
+      ".inst 0xa0438351  // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
       ".inst 0xc153be33  // udot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n"
-      "addvl x25, x25, #16\n"
+      "addvl x26, x26, #16\n"
       "40:"  // Width 4: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 41f\n"
       "udot z26.s, z3.b, z24.b\n"
       "41:"  // Width 4: Multiply loop: unique 8: skip row sum
       "tbnz %x[flags], #31, 42f\n"
-      "add x20, %x[qp], %[b_offset]\n"
-      "mov x19, #0x4\n"
-      "ld1rw { z10.s }, p2/Z, [x20]\n"
+      "add x21, %x[qp], %[b_offset]\n"
+      "mov x20, #0x4\n"
+      "ld1rw { z10.s }, p2/Z, [x21]\n"
       "neg z10.s, p2/M, z10.s\n"
-      "whilelt p0.s, XZR, x19\n"
+      "whilelt p0.s, XZR, x20\n"
       "uaddv d26, p0, z26.s\n"
       "mov z26.s, z26.s[0]\n"
       "mul z26.s, p2/M, z26.s, z10.s\n"
       "orr %x[flags], %x[flags], #0x80000000\n"
       "42:"  // Width 4: skip row sum fixup
       ".inst 0xc0904b40  // addha za0.s, p2/M, p2/M, z26.s\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[per_layer_right_shift]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       ".inst 0xc0904b41  // addha za1.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[c_offset]\n"
+      "add x21, %x[qp], %[minval]\n"
       ".inst 0xc0904b42  // addha za2.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[maxval]\n"
       ".inst 0xc0904b43  // addha za3.s, p2/M, p2/M, z26.s\n"
-      "ld1rw { z21.s }, p2/Z, [x20]\n"
+      "ld1rw { z21.s }, p2/Z, [x21]\n"
       ".inst 0xc0062c1c  // mova { z28.d-z31.d }, za.d[x9, #0]\n"
       ".inst 0xc1a5ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n"
       ".inst 0xc0062c2c  // mova { z12.d-z15.d }, za.d[x9, #1]\n"
@@ -630,7 +630,7 @@ void sme2_gemv_u8qa_dot_16VL (
       ".inst 0xc0062c68  // mova { z8.d-z11.d }, za.d[x9, #3]\n"
       ".inst 0xc1a5ac08  // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z5.s\n"
       ".inst 0xc1a4aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
       ".inst 0xc1a4aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n"
       ".inst 0xc1a4aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n"
       ".inst 0xc1a4aa28  // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n"
@@ -651,24 +651,24 @@ void sme2_gemv_u8qa_dot_16VL (
       "uzp1 z8.h, z8.h, z9.h\n"
       "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p2, [x24]\n"
+      "st1b { z28.b }, p2, [x25]\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p2, [x24, #1, MUL VL]\n"
+      "st1b { z12.b }, p2, [x25, #1, MUL VL]\n"
       "uzp1 z0.b, z0.b, z1.b\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z0.b }, p2, [x24, #2, MUL VL]\n"
-      "st1b { z8.b }, p1, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
+      "st1b { z0.b }, p2, [x25, #2, MUL VL]\n"
+      "st1b { z8.b }, p1, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
       "43:"  // Width 4: Output done
-      "subs x26, x26, #0x4\n"
-      "sub %x[N], %x[N], x27, LSL #2\n"
+      "subs x27, x27, #0x4\n"
+      "sub %x[N], %x[N], x28, LSL #2\n"
       "bgt 4b\n"
       "44:"  // Exit
       ".inst 0xd503467f  // SMSTOP\n"
       "ptrue p2.b\n"
       : [N] "+&r" (N), [flags] "+&r" (flags)
       : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [output_ptr] "r" (output_ptr), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp
index bb8cad3357..c6eb858ade 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,108 +103,108 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x14, [%x[args], %[offsetof_flags]]\n"
+      "ldr x15, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x11, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x14, #0, 2f\n"
+      "tbz x15, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      "addvl x14, x14, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w10, [%x[args], %[offsetof_M]]\n"
+      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "mov x10, #0x0\n"
       "mov x9, #0x0\n"
-      "mov x28, #0x0\n"
-      "ldr w27, [%x[args], %[offsetof_N]]\n"
-      "ldr x26, [%x[args], %[offsetof_A]]\n"
+      "ldr w28, [%x[args], %[offsetof_N]]\n"
+      "ldr x27, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x25, x26\n"
-      ".inst 0x25bb6790  // whilelt pn8.s, x28, x27, VLx4\n"
-      "tbnz x14, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x26, x27\n"
+      ".inst 0x25bc6530  // whilelt pn8.s, x9, x28, VLx4\n"
+      "tbnz x15, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z21.s, #1.0\n"
-      ".inst 0xa01cc27d  // ldnt1w { z28.s-z31.s }, p8/Z, [x19, x28, LSL #2]\n"
+      ".inst 0xa009c29d  // ldnt1w { z28.s-z31.s }, p8/Z, [x20, x9, LSL #2]\n"
       ".inst 0x809c02a0  // fmopa za0.s, p0/M, p0/M, z21.s, z28.s\n"
       ".inst 0x809d02a1  // fmopa za1.s, p0/M, p0/M, z21.s, z29.s\n"
       ".inst 0x809e02a2  // fmopa za2.s, p0/M, p0/M, z21.s, z30.s\n"
       ".inst 0x809f02a3  // fmopa za3.s, p0/M, p0/M, z21.s, z31.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x28\n"
       "mov x20, x9\n"
-      "incw x19, ALL, MUL #4\n"
-      "incw x20\n"
-      "cmp x19, x27\n"
-      "csel x20, x9, x20, LT\n"
-      "mov x19, x14\n"
-      "bfm x14, XZR, #0x0, #0x0  // bfc x14, #0x0, #0x1\n"
-      "cmp x20, x10\n"
-      "csel x14, x19, x14, LT\n"
+      "mov x21, x10\n"
+      "incw x20, ALL, MUL #4\n"
+      "incw x21\n"
+      "cmp x20, x28\n"
+      "csel x21, x10, x21, LT\n"
+      "mov x20, x15\n"
+      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
+      "cmp x21, x11\n"
+      "csel x15, x20, x15, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x1\n"
-      "lsr x19, x19, #0x1\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x28, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      "ld1h { z0.h }, p0/Z, [x25]\n"
-      ".inst 0xa140a6db  // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x22]\n"
-      "ld1h { z13.h }, p0/Z, [x25, #1, MUL VL]\n"
-      ".inst 0xa141a6ca  // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      "ld1h { z12.h }, p0/Z, [x25, #2, MUL VL]\n"
-      ".inst 0xa142a6cb  // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
-      "ld1h { z26.h }, p0/Z, [x25, #3, MUL VL]\n"
-      "addvl x25, x25, #4\n"
-      ".inst 0xa143a6d8  // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x1\n"
+      "lsr x20, x20, #0x1\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x9, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
+      ".inst 0xa140a6fb  // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n"
+      "ld1h { z13.h }, p0/Z, [x26, #1, MUL VL]\n"
+      ".inst 0xa141a6ea  // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      "ld1h { z12.h }, p0/Z, [x26, #2, MUL VL]\n"
+      ".inst 0xa142a6eb  // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      "ld1h { z26.h }, p0/Z, [x26, #3, MUL VL]\n"
+      "addvl x26, x26, #4\n"
+      ".inst 0xa143a6f8  // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x81930000  // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0x81970001  // bfmopa za1.s, p0/M, p0/M, z0.h, z23.h\n"
       ".inst 0x819b0002  // bfmopa za2.s, p0/M, p0/M, z0.h, z27.h\n"
       ".inst 0x819f0003  // bfmopa za3.s, p0/M, p0/M, z0.h, z31.h\n"
-      "ld1h { z0.h }, p0/Z, [x25]\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
       ".inst 0x818201a0  // bfmopa za0.s, p0/M, p0/M, z13.h, z2.h\n"
-      ".inst 0xa140a6db  // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x22]\n"
+      ".inst 0xa140a6fb  // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n"
       ".inst 0x818601a1  // bfmopa za1.s, p0/M, p0/M, z13.h, z6.h\n"
       ".inst 0x818a01a2  // bfmopa za2.s, p0/M, p0/M, z13.h, z10.h\n"
       ".inst 0x818e01a3  // bfmopa za3.s, p0/M, p0/M, z13.h, z14.h\n"
-      "ld1h { z13.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z13.h }, p0/Z, [x26, #1, MUL VL]\n"
       ".inst 0x81830180  // bfmopa za0.s, p0/M, p0/M, z12.h, z3.h\n"
-      ".inst 0xa141a6ca  // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa141a6ea  // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0x81870181  // bfmopa za1.s, p0/M, p0/M, z12.h, z7.h\n"
       ".inst 0x818b0182  // bfmopa za2.s, p0/M, p0/M, z12.h, z11.h\n"
       ".inst 0x818f0183  // bfmopa za3.s, p0/M, p0/M, z12.h, z15.h\n"
-      "ld1h { z12.h }, p0/Z, [x25, #2, MUL VL]\n"
-      ".inst 0xa142a6cb  // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
+      "ld1h { z12.h }, p0/Z, [x26, #2, MUL VL]\n"
+      ".inst 0xa142a6eb  // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
       ".inst 0x81900340  // bfmopa za0.s, p0/M, p0/M, z26.h, z16.h\n"
       ".inst 0x81940341  // bfmopa za1.s, p0/M, p0/M, z26.h, z20.h\n"
       ".inst 0x81980342  // bfmopa za2.s, p0/M, p0/M, z26.h, z24.h\n"
       ".inst 0x819c0343  // bfmopa za3.s, p0/M, p0/M, z26.h, z28.h\n"
-      "ld1h { z26.h }, p0/Z, [x25, #3, MUL VL]\n"
-      "addvl x25, x25, #4\n"
-      ".inst 0xa143a6d8  // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ld1h { z26.h }, p0/Z, [x26, #3, MUL VL]\n"
+      "addvl x26, x26, #4\n"
+      ".inst 0xa143a6f8  // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x81930000  // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n"
@@ -224,123 +224,123 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con
       ".inst 0x81980342  // bfmopa za2.s, p0/M, p0/M, z26.h, z24.h\n"
       ".inst 0x819c0343  // bfmopa za3.s, p0/M, p0/M, z26.h, z28.h\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      "ld1h { z0.h }, p0/Z, [x25]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x25, x25, #1\n"
-      ".inst 0xa140a6d3  // ld1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #4\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x26, x26, #1\n"
+      ".inst 0xa140a6f3  // ld1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #4\n"
       ".inst 0x81930000  // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n"
       ".inst 0x81970001  // bfmopa za1.s, p0/M, p0/M, z0.h, z23.h\n"
       ".inst 0x819b0002  // bfmopa za2.s, p0/M, p0/M, z0.h, z27.h\n"
       ".inst 0x819f0003  // bfmopa za3.s, p0/M, p0/M, z0.h, z31.h\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x14, #1, 14f\n"
-      "tbz x14, #0, 12f\n"
+      "tbz x15, #1, 14f\n"
+      "tbz x15, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0860418  // mova { z24.s-z27.s }, za0h.s[x12]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc0860434  // mova { z20.s-z23.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840781  // mova za1h.s[x12], { z28.s-z31.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840582  // mova za2h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa043c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c578  // st1w { z24.s-z27.s }, pn9.b, [x11]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5b8  // st1w { z24.s-z27.s }, pn9.b, [x13]\n"
+      "addvl x14, x14, #16\n"
+      ".inst 0xa061c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa063c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n"
       "addvl x13, x13, #16\n"
-      ".inst 0xa061c574  // st1w { z20.s-z23.s }, pn9.b, [x11, #0x4, MUL VL]\n"
-      ".inst 0xa062c57c  // st1w { z28.s-z31.s }, pn9.b, [x11, #0x8, MUL VL]\n"
-      ".inst 0xa063c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n"
-      "addvl x11, x11, #16\n"
       "blt 11b\n"
       "b 24f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
-      ".inst 0xa060c56c  // st1w { z12.s-z15.s }, pn9.b, [x11]\n"
+      ".inst 0xa060c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13]\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860464  // mova { z4.s-z7.s }, za3h.s[x12]\n"
-      ".inst 0xa061c57c  // st1w { z28.s-z31.s }, pn9.b, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa061c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0x8, MUL VL]\n"
-      ".inst 0xa063c564  // st1w { z4.s-z7.s }, pn9.b, [x11, #0xc, MUL VL]\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa063c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0xc, MUL VL]\n"
+      "addvl x13, x13, #16\n"
       "blt 13b\n"
       "b 24f\n"
       "14:"  // Store to output array
-      "ldr x24, [%x[args], %[offsetof_C]]\n"
-      "add x24, x24, x28, LSL #2\n"  // C += n
-      "sub x23, x10, x9\n"
-      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x24, x9, x22, x24\n"  // C += m * ldc
-      "tbz x14, #2, 18f\n"
-      "cntw x19\n"
-      "cmp x23, x19\n"
-      "csel x21, x23, x19, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x25, [%x[args], %[offsetof_C]]\n"
+      "add x25, x25, x9, LSL #2\n"  // C += n
+      "sub x24, x11, x10\n"
+      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "tbz x15, #2, 18f\n"
+      "cntw x20\n"
+      "cmp x24, x20\n"
+      "csel x22, x24, x20, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa160c303  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa160c323  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 17f\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x23, x23, x21\n"
+      "subs x24, x24, x22\n"
       "beq 18f\n"
       "b 22f\n"
       "18:"  // Store to output array: Skip activation: End
-      "cntw x19\n"
-      "cmp x23, x19\n"
+      "cntw x20\n"
+      "cmp x24, x20\n"
       "ld1rw { z23.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x19, x23, x19, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "csel x20, x24, x20, LT\n"
+      "lsr x21, x20, #0x2\n"
       "ld1rw { z16.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
@@ -350,19 +350,19 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
       ".inst 0xc1b0cae8  // fclamp { z8.s-z11.s }, z23.s, z16.s\n"
       ".inst 0xc1b0caec  // fclamp { z12.s-z15.s }, z23.s, z16.s\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      ".inst 0xa160c303  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      ".inst 0xa160c323  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc1b0cae0  // fclamp { z0.s-z3.s }, z23.s, z16.s\n"
@@ -371,46 +371,46 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
       ".inst 0xc1b0cae8  // fclamp { z8.s-z11.s }, z23.s, z16.s\n"
       ".inst 0xc1b0caec  // fclamp { z12.s-z15.s }, z23.s, z16.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 21f\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
       "21:"  // Store to output array: Accumulator row 0 oddments: End
       "22:"  // Store to output array: End
-      "tbz x14, #0, 24f\n"
+      "tbz x15, #0, 24f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "23:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5a8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840503  // mova za3h.s[x12], { z8.s-z11.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      "addvl x14, x14, #16\n"
       "blt 23b\n"
       "24:"  // End block
-      "incw x28, ALL, MUL #4\n"
-      "cmp x28, x27\n"
+      "incw x9, ALL, MUL #4\n"
+      "cmp x9, x28\n"
       "blt 3b\n"
-      "incw x9\n"
-      "cmp x9, x10\n"
-      "mov x28, #0x0\n"
-      "mov x26, x25\n"
+      "incw x10\n"
+      "cmp x10, x11\n"
+      "mov x9, #0x0\n"
+      "mov x27, x26\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp
index a4a40ad5ff..b63f2110ff 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,108 +103,108 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840500  // mova za0h.s[x12], { z8.s-z11.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      ".inst 0x25bc4530  // whilelt pn8.s, x9, x28, VLx2\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      ".inst 0x25a94550  // whilelt pn8.s, x10, x9, VLx2\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z21.s, #1.0\n"
-      ".inst 0xa009426f  // ldnt1w { z14.s-z15.s }, p8/Z, [x19, x9, LSL #2]\n"
+      ".inst 0xa00a428f  // ldnt1w { z14.s-z15.s }, p8/Z, [x20, x10, LSL #2]\n"
       ".inst 0x808e02a0  // fmopa za0.s, p0/M, p0/M, z21.s, z14.s\n"
       ".inst 0x808f02a1  // fmopa za1.s, p0/M, p0/M, z21.s, z15.s\n"
       ".inst 0x808e02a2  // fmopa za2.s, p0/M, p0/M, z21.s, z14.s\n"
       ".inst 0x808f02a3  // fmopa za3.s, p0/M, p0/M, z21.s, z15.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19, ALL, MUL #2\n"
+      "mov x21, x11\n"
       "incw x20, ALL, MUL #2\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "incw x21, ALL, MUL #2\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x1\n"
-      "lsr x19, x19, #0x1\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa1402747  // ld1h { z7.h, z15.h }, pn9.b/Z, [x26]\n"
-      ".inst 0xa14026df  // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x22]\n"
-      ".inst 0xa0412748  // ld1h { z8.h-z9.h }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
-      ".inst 0xa04126c3  // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
-      ".inst 0xa1422752  // ld1h { z18.h, z26.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa04226d1  // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      ".inst 0xa1432756  // ld1h { z22.h, z30.h }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14326cc  // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x1\n"
+      "lsr x20, x20, #0x1\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa1402767  // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n"
+      ".inst 0xa14026ff  // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x23]\n"
+      ".inst 0xa0412768  // ld1h { z8.h-z9.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
+      ".inst 0xa04126e3  // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
+      ".inst 0xa1422772  // ld1h { z18.h, z26.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04226f1  // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa1432776  // ld1h { z22.h, z30.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14326ec  // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x819700e0  // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0x819f00e1  // bfmopa za1.s, p0/M, p0/M, z7.h, z31.h\n"
       ".inst 0x819701e2  // bfmopa za2.s, p0/M, p0/M, z15.h, z23.h\n"
       ".inst 0x819f01e3  // bfmopa za3.s, p0/M, p0/M, z15.h, z31.h\n"
-      ".inst 0xa1402747  // ld1h { z7.h, z15.h }, pn9.b/Z, [x26]\n"
+      ".inst 0xa1402767  // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n"
       ".inst 0x81820100  // bfmopa za0.s, p0/M, p0/M, z8.h, z2.h\n"
-      ".inst 0xa14026df  // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x22]\n"
+      ".inst 0xa14026ff  // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x23]\n"
       ".inst 0x81830101  // bfmopa za1.s, p0/M, p0/M, z8.h, z3.h\n"
       ".inst 0x81820122  // bfmopa za2.s, p0/M, p0/M, z9.h, z2.h\n"
       ".inst 0x81830123  // bfmopa za3.s, p0/M, p0/M, z9.h, z3.h\n"
-      ".inst 0xa0412748  // ld1h { z8.h-z9.h }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
+      ".inst 0xa0412768  // ld1h { z8.h-z9.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
       ".inst 0x81900240  // bfmopa za0.s, p0/M, p0/M, z18.h, z16.h\n"
-      ".inst 0xa04126c3  // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
+      ".inst 0xa04126e3  // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
       ".inst 0x81910241  // bfmopa za1.s, p0/M, p0/M, z18.h, z17.h\n"
       ".inst 0x81900342  // bfmopa za2.s, p0/M, p0/M, z26.h, z16.h\n"
       ".inst 0x81910343  // bfmopa za3.s, p0/M, p0/M, z26.h, z17.h\n"
-      ".inst 0xa1422752  // ld1h { z18.h, z26.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa04226d1  // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa1422772  // ld1h { z18.h, z26.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04226f1  // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0x818402c0  // bfmopa za0.s, p0/M, p0/M, z22.h, z4.h\n"
       ".inst 0x818c02c1  // bfmopa za1.s, p0/M, p0/M, z22.h, z12.h\n"
       ".inst 0x818403c2  // bfmopa za2.s, p0/M, p0/M, z30.h, z4.h\n"
       ".inst 0x818c03c3  // bfmopa za3.s, p0/M, p0/M, z30.h, z12.h\n"
-      ".inst 0xa1432756  // ld1h { z22.h, z30.h }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14326cc  // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      ".inst 0xa1432776  // ld1h { z22.h, z30.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14326ec  // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x819700e0  // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n"
@@ -224,259 +224,259 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con
       ".inst 0x818403c2  // bfmopa za2.s, p0/M, p0/M, z30.h, z4.h\n"
       ".inst 0x818c03c3  // bfmopa za3.s, p0/M, p0/M, z30.h, z12.h\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa1402747  // ld1h { z7.h, z15.h }, pn9.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #2\n"
-      ".inst 0xa14026d7  // ld1h { z23.h, z31.h }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #2\n"
+      ".inst 0xa1402767  // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #2\n"
+      ".inst 0xa14026f7  // ld1h { z23.h, z31.h }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #2\n"
       ".inst 0x819700e0  // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n"
       ".inst 0x819f00e1  // bfmopa za1.s, p0/M, p0/M, z7.h, z31.h\n"
       ".inst 0x819701e2  // bfmopa za2.s, p0/M, p0/M, z15.h, z23.h\n"
       ".inst 0x819f01e3  // bfmopa za3.s, p0/M, p0/M, z15.h, z31.h\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840401  // mova za1h.s[x12], { z0.s-z3.s }\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860468  // mova { z8.s-z11.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840702  // mova za2h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa043c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840703  // mova za3h.s[x12], { z24.s-z27.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c5d4  // st1w { z20.s-z23.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 30f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      ".inst 0xa060c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13]\n"
+      ".inst 0xa060c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14]\n"
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
-      ".inst 0xa061c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a0  // st1w { z0.s-z3.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5c4  // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c0  // st1w { z0.s-z3.s }, pn9.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 30f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "sub x24, x11, x10\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
-      "tbz x15, #2, 21f\n"
-      "cntw x22\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "sub x25, x13, x11\n"
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
+      "tbz x16, #2, 21f\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 21f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Skip activation: Accumulator row 1 loop
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 18b\n"
       "19:"  // Store to output array: Skip activation: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc086047c  // mova { z28.s-z31.s }, za3h.s[x12]\n"
-      ".inst 0xa1604334  // st1w { z20.s, z28.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604354  // st1w { z20.s, z28.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604335  // st1w { z21.s, z29.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604355  // st1w { z21.s, z29.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      ".inst 0xa1604336  // st1w { z22.s, z30.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604356  // st1w { z22.s, z30.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "20:"  // Store to output array: Skip activation: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 21f\n"
       "b 28f\n"
       "21:"  // Store to output array: Skip activation: End
-      "cntw x22\n"
-      "cmp x24, x22\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
       "ld1rw { z21.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "ld1rw { z20.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 23f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 23f\n"
       "22:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
       ".inst 0xc1b4caa4  // fclamp { z4.s-z7.s }, z21.s, z20.s\n"
       ".inst 0xc1b4caac  // fclamp { z12.s-z15.s }, z21.s, z20.s\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 22b\n"
       "23:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 24f\n"
+      "cbz x20, 24f\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
       ".inst 0xc1b4caa0  // fclamp { z0.s-z3.s }, z21.s, z20.s\n"
       ".inst 0xc1b4caa8  // fclamp { z8.s-z11.s }, z21.s, z20.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604320  // st1w { z0.s, z8.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604340  // st1w { z0.s, z8.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604321  // st1w { z1.s, z9.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604341  // st1w { z1.s, z9.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      ".inst 0xa1604322  // st1w { z2.s, z10.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604342  // st1w { z2.s, z10.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "24:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 26f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 26f\n"
       "25:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
       ".inst 0xc1b4cab0  // fclamp { z16.s-z19.s }, z21.s, z20.s\n"
       ".inst 0xc1b4cab8  // fclamp { z24.s-z27.s }, z21.s, z20.s\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604333  // st1w { z19.s, z27.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604353  // st1w { z19.s, z27.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 25b\n"
       "26:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 27f\n"
+      "cbz x20, 27f\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
       ".inst 0xc1b4cab0  // fclamp { z16.s-z19.s }, z21.s, z20.s\n"
       ".inst 0xc1b4cab8  // fclamp { z24.s-z27.s }, z21.s, z20.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
       "27:"  // Store to output array: Accumulator row 1 oddments: End
       "28:"  // Store to output array: End
-      "tbz x15, #0, 30f\n"
+      "tbz x16, #0, 30f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "29:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840503  // mova za3h.s[x12], { z8.s-z11.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 29b\n"
       "30:"  // End block
-      "incw x9, ALL, MUL #2\n"
-      "cmp x9, x28\n"
-      "blt 3b\n"
       "incw x10, ALL, MUL #2\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "cmp x10, x9\n"
+      "blt 3b\n"
+      "incw x11, ALL, MUL #2\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp
index 798a3cb470..a51b3db4b0 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,108 +103,108 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840480  // mova za0h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa041c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa042c1c0  // ld1w { z0.s-z3.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e0  // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      "whilelt p0.s, x9, x28\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      "whilelt p0.s, x10, x9\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z8.s, #1.0\n"
-      "ldnt1w { z27.s }, p0/Z, [x19, x9, LSL #2]\n"
+      "ldnt1w { z27.s }, p0/Z, [x20, x10, LSL #2]\n"
       ".inst 0x809b2500  // fmopa za0.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2501  // fmopa za1.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2502  // fmopa za2.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2503  // fmopa za3.s, p1/M, p1/M, z8.s, z27.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19\n"
-      "incw x20, ALL, MUL #4\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "mov x21, x11\n"
+      "incw x20\n"
+      "incw x21, ALL, MUL #4\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x1\n"
-      "lsr x19, x19, #0x1\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa040a344  // ld1h { z4.h-z7.h }, pn8.b/Z, [x26]\n"
-      "ldnt1h { z29.h }, p1/Z, [x22]\n"
-      ".inst 0xa041a34c  // ld1h { z12.h-z15.h }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
-      "ldnt1h { z23.h }, p1/Z, [x22, #1, MUL VL]\n"
-      ".inst 0xa042a340  // ld1h { z0.h-z3.h }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1h { z21.h }, p1/Z, [x22, #2, MUL VL]\n"
-      ".inst 0xa143a352  // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x1\n"
+      "lsr x20, x20, #0x1\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa040a364  // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n"
+      "ldnt1h { z29.h }, p1/Z, [x23]\n"
+      ".inst 0xa041a36c  // ld1h { z12.h-z15.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
+      "ldnt1h { z23.h }, p1/Z, [x23, #1, MUL VL]\n"
+      ".inst 0xa042a360  // ld1h { z0.h-z3.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1h { z21.h }, p1/Z, [x23, #2, MUL VL]\n"
+      ".inst 0xa143a372  // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x819d2480  // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0x819d24a1  // bfmopa za1.s, p1/M, p1/M, z5.h, z29.h\n"
       ".inst 0x819d24c2  // bfmopa za2.s, p1/M, p1/M, z6.h, z29.h\n"
       ".inst 0x819d24e3  // bfmopa za3.s, p1/M, p1/M, z7.h, z29.h\n"
-      ".inst 0xa040a344  // ld1h { z4.h-z7.h }, pn8.b/Z, [x26]\n"
+      ".inst 0xa040a364  // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n"
       ".inst 0x81972580  // bfmopa za0.s, p1/M, p1/M, z12.h, z23.h\n"
-      "ldnt1h { z29.h }, p1/Z, [x22]\n"
+      "ldnt1h { z29.h }, p1/Z, [x23]\n"
       ".inst 0x819725a1  // bfmopa za1.s, p1/M, p1/M, z13.h, z23.h\n"
       ".inst 0x819725c2  // bfmopa za2.s, p1/M, p1/M, z14.h, z23.h\n"
       ".inst 0x819725e3  // bfmopa za3.s, p1/M, p1/M, z15.h, z23.h\n"
-      ".inst 0xa041a34c  // ld1h { z12.h-z15.h }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
+      ".inst 0xa041a36c  // ld1h { z12.h-z15.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
       ".inst 0x81952400  // bfmopa za0.s, p1/M, p1/M, z0.h, z21.h\n"
-      "ldnt1h { z23.h }, p1/Z, [x22, #1, MUL VL]\n"
+      "ldnt1h { z23.h }, p1/Z, [x23, #1, MUL VL]\n"
       ".inst 0x81952421  // bfmopa za1.s, p1/M, p1/M, z1.h, z21.h\n"
       ".inst 0x81952442  // bfmopa za2.s, p1/M, p1/M, z2.h, z21.h\n"
       ".inst 0x81952463  // bfmopa za3.s, p1/M, p1/M, z3.h, z21.h\n"
-      ".inst 0xa042a340  // ld1h { z0.h-z3.h }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1h { z21.h }, p1/Z, [x22, #2, MUL VL]\n"
+      ".inst 0xa042a360  // ld1h { z0.h-z3.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1h { z21.h }, p1/Z, [x23, #2, MUL VL]\n"
       ".inst 0x819b2640  // bfmopa za0.s, p1/M, p1/M, z18.h, z27.h\n"
       ".inst 0x819b26c1  // bfmopa za1.s, p1/M, p1/M, z22.h, z27.h\n"
       ".inst 0x819b2742  // bfmopa za2.s, p1/M, p1/M, z26.h, z27.h\n"
       ".inst 0x819b27c3  // bfmopa za3.s, p1/M, p1/M, z30.h, z27.h\n"
-      ".inst 0xa143a352  // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      ".inst 0xa143a372  // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x819d2480  // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n"
@@ -224,391 +224,391 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con
       ".inst 0x819b2742  // bfmopa za2.s, p1/M, p1/M, z26.h, z27.h\n"
       ".inst 0x819b27c3  // bfmopa za3.s, p1/M, p1/M, z30.h, z27.h\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa040a344  // ld1h { z4.h-z7.h }, pn8.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #4\n"
-      "ld1h { z29.h }, p1/Z, [x22]\n"
-      "addvl x22, x22, #1\n"
+      ".inst 0xa040a364  // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #4\n"
+      "ld1h { z29.h }, p1/Z, [x23]\n"
+      "addvl x23, x23, #1\n"
       ".inst 0x819d2480  // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n"
       ".inst 0x819d24a1  // bfmopa za1.s, p1/M, p1/M, z5.h, z29.h\n"
       ".inst 0x819d24c2  // bfmopa za2.s, p1/M, p1/M, z6.h, z29.h\n"
       ".inst 0x819d24e3  // bfmopa za3.s, p1/M, p1/M, z7.h, z29.h\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c1c8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0860418  // mova { z24.s-z27.s }, za0h.s[x12]\n"
       ".inst 0xc0840500  // mova za0h.s[x12], { z8.s-z11.s }\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      ".inst 0xa041c1cc  // ld1w { z12.s-z15.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1ec  // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
-      ".inst 0xa042c1c8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840502  // mova za2h.s[x12], { z8.s-z11.s }\n"
-      ".inst 0xa043c1dc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1fc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c1c4  // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c1a4  // st1w { z4.s-z7.s }, pn8.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 42f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc0860438  // mova { z24.s-z27.s }, za1h.s[x12]\n"
-      ".inst 0xa060c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13]\n"
+      ".inst 0xa060c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14]\n"
       ".inst 0xc0860440  // mova { z0.s-z3.s }, za2h.s[x12]\n"
       ".inst 0xc0860468  // mova { z8.s-z11.s }, za3h.s[x12]\n"
-      ".inst 0xa061c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 42f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "sub x24, x11, x10\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
-      "tbz x15, #2, 27f\n"
-      "cntw x22\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "sub x25, x13, x11\n"
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
+      "tbz x16, #2, 27f\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Skip activation: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 18b\n"
       "19:"  // Store to output array: Skip activation: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "20:"  // Store to output array: Skip activation: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 22f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 22f\n"
       "21:"  // Store to output array: Skip activation: Accumulator row 2 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 21b\n"
       "22:"  // Store to output array: Skip activation: Accumulator row 2 oddments
-      "cbz x19, 23f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 23f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 23f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 23f\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "23:"  // Store to output array: Skip activation: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 25f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 25f\n"
       "24:"  // Store to output array: Skip activation: Accumulator row 3 loop
       ".inst 0xc0860464  // mova { z4.s-z7.s }, za3h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z7.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z7.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 24b\n"
       "25:"  // Store to output array: Skip activation: Accumulator row 3 oddments
-      "cbz x19, 26f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 26f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      "st1w { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 26f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 26f\n"
-      "st1w { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "26:"  // Store to output array: Skip activation: Accumulator row 3 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
       "b 40f\n"
       "27:"  // Store to output array: Skip activation: End
-      "cntw x22\n"
-      "cmp x24, x22\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
       "ld1rw { z25.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "ld1rw { z24.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 29f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 29f\n"
       "28:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860414  // mova { z20.s-z23.s }, za0h.s[x12]\n"
       ".inst 0xc1b8cb34  // fclamp { z20.s-z23.s }, z25.s, z24.s\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 28b\n"
       "29:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 30f\n"
+      "cbz x20, 30f\n"
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb28  // fclamp { z8.s-z11.s }, z25.s, z24.s\n"
-      "st1w { z8.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z8.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 30f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z9.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z9.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 30f\n"
-      "st1w { z10.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z10.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "30:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 32f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 32f\n"
       "31:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 31b\n"
       "32:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 33f\n"
+      "cbz x20, 33f\n"
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 33f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 33f\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "33:"  // Store to output array: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 35f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 35f\n"
       "34:"  // Store to output array: Accumulator row 2 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 34b\n"
       "35:"  // Store to output array: Accumulator row 2 oddments
-      "cbz x19, 36f\n"
+      "cbz x20, 36f\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 36f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 36f\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "36:"  // Store to output array: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 38f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 38f\n"
       "37:"  // Store to output array: Accumulator row 3 loop
       ".inst 0xc0860474  // mova { z20.s-z23.s }, za3h.s[x12]\n"
       ".inst 0xc1b8cb34  // fclamp { z20.s-z23.s }, z25.s, z24.s\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 37b\n"
       "38:"  // Store to output array: Accumulator row 3 oddments
-      "cbz x19, 39f\n"
+      "cbz x20, 39f\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 39f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 39f\n"
-      "st1w { z18.s }, p0, [x25]\n"
+      "st1w { z18.s }, p0, [x26]\n"
       "39:"  // Store to output array: Accumulator row 3 oddments: End
       "40:"  // Store to output array: End
-      "tbz x15, #0, 42f\n"
+      "tbz x16, #0, 42f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "41:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840682  // mova za2h.s[x12], { z20.s-z23.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 41b\n"
       "42:"  // End block
-      "incw x9\n"
-      "cmp x9, x28\n"
+      "incw x10\n"
+      "cmp x10, x9\n"
       "blt 3b\n"
-      "incw x10, ALL, MUL #4\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "incw x11, ALL, MUL #4\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp
index 4f6d9a3d98..dd99387c5e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,106 +103,106 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x14, [%x[args], %[offsetof_flags]]\n"
+      "ldr x15, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x11, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x14, #0, 2f\n"
+      "tbz x15, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      "addvl x14, x14, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w10, [%x[args], %[offsetof_M]]\n"
+      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "mov x10, #0x0\n"
       "mov x9, #0x0\n"
-      "mov x28, #0x0\n"
-      "ldr w27, [%x[args], %[offsetof_N]]\n"
-      "ldr x26, [%x[args], %[offsetof_A]]\n"
+      "ldr w28, [%x[args], %[offsetof_N]]\n"
+      "ldr x27, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x25, x26\n"
-      ".inst 0x25bb6790  // whilelt pn8.s, x28, x27, VLx4\n"
-      "tbnz x14, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x26, x27\n"
+      ".inst 0x25bc6530  // whilelt pn8.s, x9, x28, VLx4\n"
+      "tbnz x15, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z21.s, #1.0\n"
-      ".inst 0xa01cc27d  // ldnt1w { z28.s-z31.s }, p8/Z, [x19, x28, LSL #2]\n"
+      ".inst 0xa009c29d  // ldnt1w { z28.s-z31.s }, p8/Z, [x20, x9, LSL #2]\n"
       ".inst 0x809c02a0  // fmopa za0.s, p0/M, p0/M, z21.s, z28.s\n"
       ".inst 0x809d02a1  // fmopa za1.s, p0/M, p0/M, z21.s, z29.s\n"
       ".inst 0x809e02a2  // fmopa za2.s, p0/M, p0/M, z21.s, z30.s\n"
       ".inst 0x809f02a3  // fmopa za3.s, p0/M, p0/M, z21.s, z31.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x28\n"
       "mov x20, x9\n"
-      "incw x19, ALL, MUL #4\n"
-      "incw x20\n"
-      "cmp x19, x27\n"
-      "csel x20, x9, x20, LT\n"
-      "mov x19, x14\n"
-      "bfm x14, XZR, #0x0, #0x0  // bfc x14, #0x0, #0x1\n"
-      "cmp x20, x10\n"
-      "csel x14, x19, x14, LT\n"
+      "mov x21, x10\n"
+      "incw x20, ALL, MUL #4\n"
+      "incw x21\n"
+      "cmp x20, x28\n"
+      "csel x21, x10, x21, LT\n"
+      "mov x20, x15\n"
+      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
+      "cmp x21, x11\n"
+      "csel x15, x20, x15, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "lsr x22, x19, #0x2\n"
-      "and x21, x19, #0x3\n"
-      "ldr x20, [%x[args], %[offsetof_B]]\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x20, x28, x19, x20\n"  // bptr = B + n * kstride_bytes
-      "cbz x22, 8f\n"
-      "subs x22, x22, #0x1\n"
-      "ld1w { z0.s }, p0/Z, [x25]\n"
-      ".inst 0xa140c69b  // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x20]\n"
-      "ld1w { z13.s }, p0/Z, [x25, #1, MUL VL]\n"
-      ".inst 0xa141c68a  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x20, #0x4, MUL VL]\n"
-      "ld1w { z12.s }, p0/Z, [x25, #2, MUL VL]\n"
-      ".inst 0xa142c68b  // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x20, #0x8, MUL VL]\n"
-      "ld1w { z26.s }, p0/Z, [x25, #3, MUL VL]\n"
-      "addvl x25, x25, #4\n"
-      ".inst 0xa143c698  // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x20, #0xc, MUL VL]\n"
-      "addvl x20, x20, #16\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "lsr x23, x20, #0x2\n"
+      "and x22, x20, #0x3\n"
+      "ldr x21, [%x[args], %[offsetof_B]]\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x21, x9, x20, x21\n"  // bptr = B + n * kstride_bytes
+      "cbz x23, 8f\n"
+      "subs x23, x23, #0x1\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
+      ".inst 0xa140c6bb  // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n"
+      "ld1w { z13.s }, p0/Z, [x26, #1, MUL VL]\n"
+      ".inst 0xa141c6aa  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n"
+      "ld1w { z12.s }, p0/Z, [x26, #2, MUL VL]\n"
+      ".inst 0xa142c6ab  // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n"
+      "ld1w { z26.s }, p0/Z, [x26, #3, MUL VL]\n"
+      "addvl x26, x26, #4\n"
+      ".inst 0xa143c6b8  // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n"
+      "addvl x21, x21, #16\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x80930000  // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       ".inst 0x80970001  // fmopa za1.s, p0/M, p0/M, z0.s, z23.s\n"
       ".inst 0x809b0002  // fmopa za2.s, p0/M, p0/M, z0.s, z27.s\n"
       ".inst 0x809f0003  // fmopa za3.s, p0/M, p0/M, z0.s, z31.s\n"
-      "ld1w { z0.s }, p0/Z, [x25]\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
       ".inst 0x808201a0  // fmopa za0.s, p0/M, p0/M, z13.s, z2.s\n"
-      ".inst 0xa140c69b  // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x20]\n"
+      ".inst 0xa140c6bb  // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n"
       ".inst 0x808601a1  // fmopa za1.s, p0/M, p0/M, z13.s, z6.s\n"
       ".inst 0x808a01a2  // fmopa za2.s, p0/M, p0/M, z13.s, z10.s\n"
       ".inst 0x808e01a3  // fmopa za3.s, p0/M, p0/M, z13.s, z14.s\n"
-      "ld1w { z13.s }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z13.s }, p0/Z, [x26, #1, MUL VL]\n"
       ".inst 0x80830180  // fmopa za0.s, p0/M, p0/M, z12.s, z3.s\n"
-      ".inst 0xa141c68a  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x20, #0x4, MUL VL]\n"
+      ".inst 0xa141c6aa  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n"
       ".inst 0x80870181  // fmopa za1.s, p0/M, p0/M, z12.s, z7.s\n"
       ".inst 0x808b0182  // fmopa za2.s, p0/M, p0/M, z12.s, z11.s\n"
       ".inst 0x808f0183  // fmopa za3.s, p0/M, p0/M, z12.s, z15.s\n"
-      "ld1w { z12.s }, p0/Z, [x25, #2, MUL VL]\n"
-      ".inst 0xa142c68b  // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x20, #0x8, MUL VL]\n"
+      "ld1w { z12.s }, p0/Z, [x26, #2, MUL VL]\n"
+      ".inst 0xa142c6ab  // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n"
       ".inst 0x80900340  // fmopa za0.s, p0/M, p0/M, z26.s, z16.s\n"
       ".inst 0x80940341  // fmopa za1.s, p0/M, p0/M, z26.s, z20.s\n"
       ".inst 0x80980342  // fmopa za2.s, p0/M, p0/M, z26.s, z24.s\n"
       ".inst 0x809c0343  // fmopa za3.s, p0/M, p0/M, z26.s, z28.s\n"
-      "ld1w { z26.s }, p0/Z, [x25, #3, MUL VL]\n"
-      "addvl x25, x25, #4\n"
-      ".inst 0xa143c698  // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x20, #0xc, MUL VL]\n"
-      "addvl x20, x20, #16\n"
+      "ld1w { z26.s }, p0/Z, [x26, #3, MUL VL]\n"
+      "addvl x26, x26, #4\n"
+      ".inst 0xa143c6b8  // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n"
+      "addvl x21, x21, #16\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x80930000  // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n"
@@ -222,123 +222,123 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa
       ".inst 0x80980342  // fmopa za2.s, p0/M, p0/M, z26.s, z24.s\n"
       ".inst 0x809c0343  // fmopa za3.s, p0/M, p0/M, z26.s, z28.s\n"
       "8:"  // K oddments
-      "cbz x21, 10f\n"
+      "cbz x22, 10f\n"
       "9:"  // K oddments: Loop
-      "ld1w { z0.s }, p0/Z, [x25]\n"
-      "subs x21, x21, #0x1\n"
-      "addvl x25, x25, #1\n"
-      ".inst 0xa140c693  // ld1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x20]\n"
-      "addvl x20, x20, #4\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
+      "subs x22, x22, #0x1\n"
+      "addvl x26, x26, #1\n"
+      ".inst 0xa140c6b3  // ld1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n"
+      "addvl x21, x21, #4\n"
       ".inst 0x80930000  // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n"
       ".inst 0x80970001  // fmopa za1.s, p0/M, p0/M, z0.s, z23.s\n"
       ".inst 0x809b0002  // fmopa za2.s, p0/M, p0/M, z0.s, z27.s\n"
       ".inst 0x809f0003  // fmopa za3.s, p0/M, p0/M, z0.s, z31.s\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x14, #1, 14f\n"
-      "tbz x14, #0, 12f\n"
+      "tbz x15, #1, 14f\n"
+      "tbz x15, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0860418  // mova { z24.s-z27.s }, za0h.s[x12]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc0860434  // mova { z20.s-z23.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840781  // mova za1h.s[x12], { z28.s-z31.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840582  // mova za2h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa043c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c578  // st1w { z24.s-z27.s }, pn9.b, [x11]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5b8  // st1w { z24.s-z27.s }, pn9.b, [x13]\n"
+      "addvl x14, x14, #16\n"
+      ".inst 0xa061c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa063c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n"
       "addvl x13, x13, #16\n"
-      ".inst 0xa061c574  // st1w { z20.s-z23.s }, pn9.b, [x11, #0x4, MUL VL]\n"
-      ".inst 0xa062c57c  // st1w { z28.s-z31.s }, pn9.b, [x11, #0x8, MUL VL]\n"
-      ".inst 0xa063c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n"
-      "addvl x11, x11, #16\n"
       "blt 11b\n"
       "b 24f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
-      ".inst 0xa060c56c  // st1w { z12.s-z15.s }, pn9.b, [x11]\n"
+      ".inst 0xa060c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13]\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860464  // mova { z4.s-z7.s }, za3h.s[x12]\n"
-      ".inst 0xa061c57c  // st1w { z28.s-z31.s }, pn9.b, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa061c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0x8, MUL VL]\n"
-      ".inst 0xa063c564  // st1w { z4.s-z7.s }, pn9.b, [x11, #0xc, MUL VL]\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa063c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0xc, MUL VL]\n"
+      "addvl x13, x13, #16\n"
       "blt 13b\n"
       "b 24f\n"
       "14:"  // Store to output array
-      "ldr x24, [%x[args], %[offsetof_C]]\n"
-      "add x24, x24, x28, LSL #2\n"  // C += n
-      "sub x23, x10, x9\n"
-      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x24, x9, x22, x24\n"  // C += m * ldc
-      "tbz x14, #2, 18f\n"
-      "cntw x19\n"
-      "cmp x23, x19\n"
-      "csel x21, x23, x19, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x25, [%x[args], %[offsetof_C]]\n"
+      "add x25, x25, x9, LSL #2\n"  // C += n
+      "sub x24, x11, x10\n"
+      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "tbz x15, #2, 18f\n"
+      "cntw x20\n"
+      "cmp x24, x20\n"
+      "csel x22, x24, x20, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa160c303  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa160c323  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 17f\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x23, x23, x21\n"
+      "subs x24, x24, x22\n"
       "beq 18f\n"
       "b 22f\n"
       "18:"  // Store to output array: Skip activation: End
-      "cntw x19\n"
-      "cmp x23, x19\n"
+      "cntw x20\n"
+      "cmp x24, x20\n"
       "ld1rw { z23.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x19, x23, x19, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "csel x20, x24, x20, LT\n"
+      "lsr x21, x20, #0x2\n"
       "ld1rw { z16.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
@@ -348,19 +348,19 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
       ".inst 0xc1b0cae8  // fclamp { z8.s-z11.s }, z23.s, z16.s\n"
       ".inst 0xc1b0caec  // fclamp { z12.s-z15.s }, z23.s, z16.s\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
-      ".inst 0xa160c303  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
+      ".inst 0xa160c323  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc1b0cae0  // fclamp { z0.s-z3.s }, z23.s, z16.s\n"
@@ -369,46 +369,46 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
       ".inst 0xc1b0cae8  // fclamp { z8.s-z11.s }, z23.s, z16.s\n"
       ".inst 0xc1b0caec  // fclamp { z12.s-z15.s }, z23.s, z16.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c300  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c320  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c301  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x24]\n"
-      "add x24, x24, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c321  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n"
+      "add x25, x25, x23\n"
       "beq 21f\n"
-      ".inst 0xa160c302  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x24]\n"
+      ".inst 0xa160c322  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n"
       "21:"  // Store to output array: Accumulator row 0 oddments: End
       "22:"  // Store to output array: End
-      "tbz x14, #0, 24f\n"
+      "tbz x15, #0, 24f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "23:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n"
+      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
+      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5a8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
+      ".inst 0xa043c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
       ".inst 0xc0840503  // mova za3h.s[x12], { z8.s-z11.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      "addvl x14, x14, #16\n"
       "blt 23b\n"
       "24:"  // End block
-      "incw x28, ALL, MUL #4\n"
-      "cmp x28, x27\n"
+      "incw x9, ALL, MUL #4\n"
+      "cmp x9, x28\n"
       "blt 3b\n"
-      "incw x9\n"
-      "cmp x9, x10\n"
-      "mov x28, #0x0\n"
-      "mov x26, x25\n"
+      "incw x10\n"
+      "cmp x10, x11\n"
+      "mov x9, #0x0\n"
+      "mov x27, x26\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp
index 344215bfa5..87d7827c5b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,106 +103,106 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840500  // mova za0h.s[x12], { z8.s-z11.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      ".inst 0x25bc4530  // whilelt pn8.s, x9, x28, VLx2\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      ".inst 0x25a94550  // whilelt pn8.s, x10, x9, VLx2\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z21.s, #1.0\n"
-      ".inst 0xa009426f  // ldnt1w { z14.s-z15.s }, p8/Z, [x19, x9, LSL #2]\n"
+      ".inst 0xa00a428f  // ldnt1w { z14.s-z15.s }, p8/Z, [x20, x10, LSL #2]\n"
       ".inst 0x808e02a0  // fmopa za0.s, p0/M, p0/M, z21.s, z14.s\n"
       ".inst 0x808f02a1  // fmopa za1.s, p0/M, p0/M, z21.s, z15.s\n"
       ".inst 0x808e02a2  // fmopa za2.s, p0/M, p0/M, z21.s, z14.s\n"
       ".inst 0x808f02a3  // fmopa za3.s, p0/M, p0/M, z21.s, z15.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19, ALL, MUL #2\n"
+      "mov x21, x11\n"
       "incw x20, ALL, MUL #2\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "incw x21, ALL, MUL #2\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "lsr x22, x19, #0x2\n"
-      "and x21, x19, #0x3\n"
-      "ldr x20, [%x[args], %[offsetof_B]]\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x20, x9, x19, x20\n"  // bptr = B + n * kstride_bytes
-      "cbz x22, 8f\n"
-      "subs x22, x22, #0x1\n"
-      ".inst 0xa1404747  // ld1w { z7.s, z15.s }, pn9.b/Z, [x26]\n"
-      ".inst 0xa140469f  // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x20]\n"
-      ".inst 0xa0414748  // ld1w { z8.s-z9.s }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
-      ".inst 0xa0414683  // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x20, #0x2, MUL VL]\n"
-      ".inst 0xa1424752  // ld1w { z18.s, z26.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa0424691  // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x20, #0x4, MUL VL]\n"
-      ".inst 0xa1434756  // ld1w { z22.s, z30.s }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa143468c  // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x20, #0x6, MUL VL]\n"
-      "addvl x20, x20, #8\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "lsr x23, x20, #0x2\n"
+      "and x22, x20, #0x3\n"
+      "ldr x21, [%x[args], %[offsetof_B]]\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x21, x10, x20, x21\n"  // bptr = B + n * kstride_bytes
+      "cbz x23, 8f\n"
+      "subs x23, x23, #0x1\n"
+      ".inst 0xa1404767  // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n"
+      ".inst 0xa14046bf  // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x21]\n"
+      ".inst 0xa0414768  // ld1w { z8.s-z9.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
+      ".inst 0xa04146a3  // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n"
+      ".inst 0xa1424772  // ld1w { z18.s, z26.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04246b1  // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n"
+      ".inst 0xa1434776  // ld1w { z22.s, z30.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14346ac  // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n"
+      "addvl x21, x21, #8\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x809700e0  // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       ".inst 0x809f00e1  // fmopa za1.s, p0/M, p0/M, z7.s, z31.s\n"
       ".inst 0x809701e2  // fmopa za2.s, p0/M, p0/M, z15.s, z23.s\n"
       ".inst 0x809f01e3  // fmopa za3.s, p0/M, p0/M, z15.s, z31.s\n"
-      ".inst 0xa1404747  // ld1w { z7.s, z15.s }, pn9.b/Z, [x26]\n"
+      ".inst 0xa1404767  // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n"
       ".inst 0x80820100  // fmopa za0.s, p0/M, p0/M, z8.s, z2.s\n"
-      ".inst 0xa140469f  // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x20]\n"
+      ".inst 0xa14046bf  // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x21]\n"
       ".inst 0x80830101  // fmopa za1.s, p0/M, p0/M, z8.s, z3.s\n"
       ".inst 0x80820122  // fmopa za2.s, p0/M, p0/M, z9.s, z2.s\n"
       ".inst 0x80830123  // fmopa za3.s, p0/M, p0/M, z9.s, z3.s\n"
-      ".inst 0xa0414748  // ld1w { z8.s-z9.s }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
+      ".inst 0xa0414768  // ld1w { z8.s-z9.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
       ".inst 0x80900240  // fmopa za0.s, p0/M, p0/M, z18.s, z16.s\n"
-      ".inst 0xa0414683  // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x20, #0x2, MUL VL]\n"
+      ".inst 0xa04146a3  // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n"
       ".inst 0x80910241  // fmopa za1.s, p0/M, p0/M, z18.s, z17.s\n"
       ".inst 0x80900342  // fmopa za2.s, p0/M, p0/M, z26.s, z16.s\n"
       ".inst 0x80910343  // fmopa za3.s, p0/M, p0/M, z26.s, z17.s\n"
-      ".inst 0xa1424752  // ld1w { z18.s, z26.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa0424691  // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x20, #0x4, MUL VL]\n"
+      ".inst 0xa1424772  // ld1w { z18.s, z26.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04246b1  // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n"
       ".inst 0x808402c0  // fmopa za0.s, p0/M, p0/M, z22.s, z4.s\n"
       ".inst 0x808c02c1  // fmopa za1.s, p0/M, p0/M, z22.s, z12.s\n"
       ".inst 0x808403c2  // fmopa za2.s, p0/M, p0/M, z30.s, z4.s\n"
       ".inst 0x808c03c3  // fmopa za3.s, p0/M, p0/M, z30.s, z12.s\n"
-      ".inst 0xa1434756  // ld1w { z22.s, z30.s }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa143468c  // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x20, #0x6, MUL VL]\n"
-      "addvl x20, x20, #8\n"
+      ".inst 0xa1434776  // ld1w { z22.s, z30.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14346ac  // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n"
+      "addvl x21, x21, #8\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x809700e0  // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n"
@@ -222,259 +222,259 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa
       ".inst 0x808403c2  // fmopa za2.s, p0/M, p0/M, z30.s, z4.s\n"
       ".inst 0x808c03c3  // fmopa za3.s, p0/M, p0/M, z30.s, z12.s\n"
       "8:"  // K oddments
-      "cbz x21, 10f\n"
+      "cbz x22, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa1404747  // ld1w { z7.s, z15.s }, pn9.b/Z, [x26]\n"
-      "subs x21, x21, #0x1\n"
-      "addvl x26, x26, #2\n"
-      ".inst 0xa1404697  // ld1w { z23.s, z31.s }, pn9.b/Z, [x20]\n"
-      "addvl x20, x20, #2\n"
+      ".inst 0xa1404767  // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n"
+      "subs x22, x22, #0x1\n"
+      "addvl x27, x27, #2\n"
+      ".inst 0xa14046b7  // ld1w { z23.s, z31.s }, pn9.b/Z, [x21]\n"
+      "addvl x21, x21, #2\n"
       ".inst 0x809700e0  // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n"
       ".inst 0x809f00e1  // fmopa za1.s, p0/M, p0/M, z7.s, z31.s\n"
       ".inst 0x809701e2  // fmopa za2.s, p0/M, p0/M, z15.s, z23.s\n"
       ".inst 0x809f01e3  // fmopa za3.s, p0/M, p0/M, z15.s, z31.s\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840401  // mova za1h.s[x12], { z0.s-z3.s }\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860468  // mova { z8.s-z11.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840702  // mova za2h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa043c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840703  // mova za3h.s[x12], { z24.s-z27.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c5d4  // st1w { z20.s-z23.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 30f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      ".inst 0xa060c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13]\n"
+      ".inst 0xa060c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14]\n"
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
-      ".inst 0xa061c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a0  // st1w { z0.s-z3.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5c4  // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c0  // st1w { z0.s-z3.s }, pn9.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 30f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "sub x24, x11, x10\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
-      "tbz x15, #2, 21f\n"
-      "cntw x22\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "sub x25, x13, x11\n"
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
+      "tbz x16, #2, 21f\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 21f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Skip activation: Accumulator row 1 loop
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 18b\n"
       "19:"  // Store to output array: Skip activation: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc086047c  // mova { z28.s-z31.s }, za3h.s[x12]\n"
-      ".inst 0xa1604334  // st1w { z20.s, z28.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604354  // st1w { z20.s, z28.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604335  // st1w { z21.s, z29.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604355  // st1w { z21.s, z29.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      ".inst 0xa1604336  // st1w { z22.s, z30.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604356  // st1w { z22.s, z30.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "20:"  // Store to output array: Skip activation: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 21f\n"
       "b 28f\n"
       "21:"  // Store to output array: Skip activation: End
-      "cntw x22\n"
-      "cmp x24, x22\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
       "ld1rw { z21.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "ld1rw { z20.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 23f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 23f\n"
       "22:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
       ".inst 0xc1b4caa4  // fclamp { z4.s-z7.s }, z21.s, z20.s\n"
       ".inst 0xc1b4caac  // fclamp { z12.s-z15.s }, z21.s, z20.s\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604327  // st1w { z7.s, z15.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604347  // st1w { z7.s, z15.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 22b\n"
       "23:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 24f\n"
+      "cbz x20, 24f\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
       ".inst 0xc1b4caa0  // fclamp { z0.s-z3.s }, z21.s, z20.s\n"
       ".inst 0xc1b4caa8  // fclamp { z8.s-z11.s }, z21.s, z20.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604320  // st1w { z0.s, z8.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604340  // st1w { z0.s, z8.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604321  // st1w { z1.s, z9.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604341  // st1w { z1.s, z9.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      ".inst 0xa1604322  // st1w { z2.s, z10.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604342  // st1w { z2.s, z10.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "24:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 26f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 26f\n"
       "25:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
       ".inst 0xc1b4cab0  // fclamp { z16.s-z19.s }, z21.s, z20.s\n"
       ".inst 0xc1b4cab8  // fclamp { z24.s-z27.s }, z21.s, z20.s\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
-      ".inst 0xa1604333  // st1w { z19.s, z27.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
+      ".inst 0xa1604353  // st1w { z19.s, z27.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 25b\n"
       "26:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 27f\n"
+      "cbz x20, 27f\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
       ".inst 0xc1b4cab0  // fclamp { z16.s-z19.s }, z21.s, z20.s\n"
       ".inst 0xc1b4cab8  // fclamp { z24.s-z27.s }, z21.s, z20.s\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
       "27:"  // Store to output array: Accumulator row 1 oddments: End
       "28:"  // Store to output array: End
-      "tbz x15, #0, 30f\n"
+      "tbz x16, #0, 30f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "29:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840503  // mova za3h.s[x12], { z8.s-z11.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 29b\n"
       "30:"  // End block
-      "incw x9, ALL, MUL #2\n"
-      "cmp x9, x28\n"
-      "blt 3b\n"
       "incw x10, ALL, MUL #2\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "cmp x10, x9\n"
+      "blt 3b\n"
+      "incw x11, ALL, MUL #2\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp
index 5252e8140b..291a7ced5a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -103,106 +103,106 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa
   KernelArgs args(A, B, C, ldc, M, N, K, bias, act, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840480  // mova za0h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa041c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa042c1c0  // ld1w { z0.s-z3.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e0  // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      "whilelt p0.s, x9, x28\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      "whilelt p0.s, x10, x9\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "fmov z8.s, #1.0\n"
-      "ldnt1w { z27.s }, p0/Z, [x19, x9, LSL #2]\n"
+      "ldnt1w { z27.s }, p0/Z, [x20, x10, LSL #2]\n"
       ".inst 0x809b2500  // fmopa za0.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2501  // fmopa za1.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2502  // fmopa za2.s, p1/M, p1/M, z8.s, z27.s\n"
       ".inst 0x809b2503  // fmopa za3.s, p1/M, p1/M, z8.s, z27.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19\n"
-      "incw x20, ALL, MUL #4\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "mov x21, x11\n"
+      "incw x20\n"
+      "incw x21, ALL, MUL #4\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "lsr x22, x19, #0x2\n"
-      "and x21, x19, #0x3\n"
-      "ldr x20, [%x[args], %[offsetof_B]]\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x20, x9, x19, x20\n"  // bptr = B + n * kstride_bytes
-      "cbz x22, 8f\n"
-      "subs x22, x22, #0x1\n"
-      ".inst 0xa040c344  // ld1w { z4.s-z7.s }, pn8.b/Z, [x26]\n"
-      "ldnt1w { z29.s }, p1/Z, [x20]\n"
-      ".inst 0xa041c34c  // ld1w { z12.s-z15.s }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
-      "ldnt1w { z23.s }, p1/Z, [x20, #1, MUL VL]\n"
-      ".inst 0xa042c340  // ld1w { z0.s-z3.s }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1w { z21.s }, p1/Z, [x20, #2, MUL VL]\n"
-      ".inst 0xa143c352  // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
-      "addvl x20, x20, #4\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "lsr x23, x20, #0x2\n"
+      "and x22, x20, #0x3\n"
+      "ldr x21, [%x[args], %[offsetof_B]]\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x21, x10, x20, x21\n"  // bptr = B + n * kstride_bytes
+      "cbz x23, 8f\n"
+      "subs x23, x23, #0x1\n"
+      ".inst 0xa040c364  // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n"
+      "ldnt1w { z29.s }, p1/Z, [x21]\n"
+      ".inst 0xa041c36c  // ld1w { z12.s-z15.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
+      "ldnt1w { z23.s }, p1/Z, [x21, #1, MUL VL]\n"
+      ".inst 0xa042c360  // ld1w { z0.s-z3.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1w { z21.s }, p1/Z, [x21, #2, MUL VL]\n"
+      ".inst 0xa143c372  // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "addvl x21, x21, #4\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0x809d2480  // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       ".inst 0x809d24a1  // fmopa za1.s, p1/M, p1/M, z5.s, z29.s\n"
       ".inst 0x809d24c2  // fmopa za2.s, p1/M, p1/M, z6.s, z29.s\n"
       ".inst 0x809d24e3  // fmopa za3.s, p1/M, p1/M, z7.s, z29.s\n"
-      ".inst 0xa040c344  // ld1w { z4.s-z7.s }, pn8.b/Z, [x26]\n"
+      ".inst 0xa040c364  // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n"
       ".inst 0x80972580  // fmopa za0.s, p1/M, p1/M, z12.s, z23.s\n"
-      "ldnt1w { z29.s }, p1/Z, [x20]\n"
+      "ldnt1w { z29.s }, p1/Z, [x21]\n"
       ".inst 0x809725a1  // fmopa za1.s, p1/M, p1/M, z13.s, z23.s\n"
       ".inst 0x809725c2  // fmopa za2.s, p1/M, p1/M, z14.s, z23.s\n"
       ".inst 0x809725e3  // fmopa za3.s, p1/M, p1/M, z15.s, z23.s\n"
-      ".inst 0xa041c34c  // ld1w { z12.s-z15.s }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
+      ".inst 0xa041c36c  // ld1w { z12.s-z15.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
       ".inst 0x80952400  // fmopa za0.s, p1/M, p1/M, z0.s, z21.s\n"
-      "ldnt1w { z23.s }, p1/Z, [x20, #1, MUL VL]\n"
+      "ldnt1w { z23.s }, p1/Z, [x21, #1, MUL VL]\n"
       ".inst 0x80952421  // fmopa za1.s, p1/M, p1/M, z1.s, z21.s\n"
       ".inst 0x80952442  // fmopa za2.s, p1/M, p1/M, z2.s, z21.s\n"
       ".inst 0x80952463  // fmopa za3.s, p1/M, p1/M, z3.s, z21.s\n"
-      ".inst 0xa042c340  // ld1w { z0.s-z3.s }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1w { z21.s }, p1/Z, [x20, #2, MUL VL]\n"
+      ".inst 0xa042c360  // ld1w { z0.s-z3.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1w { z21.s }, p1/Z, [x21, #2, MUL VL]\n"
       ".inst 0x809b2640  // fmopa za0.s, p1/M, p1/M, z18.s, z27.s\n"
       ".inst 0x809b26c1  // fmopa za1.s, p1/M, p1/M, z22.s, z27.s\n"
       ".inst 0x809b2742  // fmopa za2.s, p1/M, p1/M, z26.s, z27.s\n"
       ".inst 0x809b27c3  // fmopa za3.s, p1/M, p1/M, z30.s, z27.s\n"
-      ".inst 0xa143c352  // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
-      "addvl x20, x20, #4\n"
+      ".inst 0xa143c372  // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "addvl x21, x21, #4\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0x809d2480  // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n"
@@ -222,391 +222,391 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa
       ".inst 0x809b2742  // fmopa za2.s, p1/M, p1/M, z26.s, z27.s\n"
       ".inst 0x809b27c3  // fmopa za3.s, p1/M, p1/M, z30.s, z27.s\n"
       "8:"  // K oddments
-      "cbz x21, 10f\n"
+      "cbz x22, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa040c344  // ld1w { z4.s-z7.s }, pn8.b/Z, [x26]\n"
-      "subs x21, x21, #0x1\n"
-      "addvl x26, x26, #4\n"
-      "ld1w { z29.s }, p1/Z, [x20]\n"
-      "addvl x20, x20, #1\n"
+      ".inst 0xa040c364  // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n"
+      "subs x22, x22, #0x1\n"
+      "addvl x27, x27, #4\n"
+      "ld1w { z29.s }, p1/Z, [x21]\n"
+      "addvl x21, x21, #1\n"
       ".inst 0x809d2480  // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n"
       ".inst 0x809d24a1  // fmopa za1.s, p1/M, p1/M, z5.s, z29.s\n"
       ".inst 0x809d24c2  // fmopa za2.s, p1/M, p1/M, z6.s, z29.s\n"
       ".inst 0x809d24e3  // fmopa za3.s, p1/M, p1/M, z7.s, z29.s\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c1c8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0860418  // mova { z24.s-z27.s }, za0h.s[x12]\n"
       ".inst 0xc0840500  // mova za0h.s[x12], { z8.s-z11.s }\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      ".inst 0xa041c1cc  // ld1w { z12.s-z15.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1ec  // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
-      ".inst 0xa042c1c8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840502  // mova za2h.s[x12], { z8.s-z11.s }\n"
-      ".inst 0xa043c1dc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1fc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c1c4  // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c1a4  // st1w { z4.s-z7.s }, pn8.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 42f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc0860438  // mova { z24.s-z27.s }, za1h.s[x12]\n"
-      ".inst 0xa060c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13]\n"
+      ".inst 0xa060c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14]\n"
       ".inst 0xc0860440  // mova { z0.s-z3.s }, za2h.s[x12]\n"
       ".inst 0xc0860468  // mova { z8.s-z11.s }, za3h.s[x12]\n"
-      ".inst 0xa061c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 42f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "sub x24, x11, x10\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
-      "tbz x15, #2, 27f\n"
-      "cntw x22\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "sub x25, x13, x11\n"
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
+      "tbz x16, #2, 27f\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Skip activation: Accumulator row 0 loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 15b\n"
       "16:"  // Store to output array: Skip activation: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 17f\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "17:"  // Store to output array: Skip activation: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Skip activation: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 18b\n"
       "19:"  // Store to output array: Skip activation: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 20f\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "20:"  // Store to output array: Skip activation: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 22f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 22f\n"
       "21:"  // Store to output array: Skip activation: Accumulator row 2 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 21b\n"
       "22:"  // Store to output array: Skip activation: Accumulator row 2 oddments
-      "cbz x19, 23f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 23f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 23f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 23f\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "23:"  // Store to output array: Skip activation: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 25f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 25f\n"
       "24:"  // Store to output array: Skip activation: Accumulator row 3 loop
       ".inst 0xc0860464  // mova { z4.s-z7.s }, za3h.s[x12]\n"
-      "st1w { z4.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z5.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z4.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z5.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z6.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z7.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z6.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z7.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 24b\n"
       "25:"  // Store to output array: Skip activation: Accumulator row 3 oddments
-      "cbz x19, 26f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 26f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      "st1w { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 26f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 26f\n"
-      "st1w { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "26:"  // Store to output array: Skip activation: Accumulator row 3 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
       "b 40f\n"
       "27:"  // Store to output array: Skip activation: End
-      "cntw x22\n"
-      "cmp x24, x22\n"
+      "cntw x23\n"
+      "cmp x25, x23\n"
       "ld1rw { z25.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "ld1rw { z24.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 29f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 29f\n"
       "28:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860414  // mova { z20.s-z23.s }, za0h.s[x12]\n"
       ".inst 0xc1b8cb34  // fclamp { z20.s-z23.s }, z25.s, z24.s\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 28b\n"
       "29:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 30f\n"
+      "cbz x20, 30f\n"
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb28  // fclamp { z8.s-z11.s }, z25.s, z24.s\n"
-      "st1w { z8.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z8.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 30f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z9.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z9.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 30f\n"
-      "st1w { z10.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z10.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "30:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 32f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 32f\n"
       "31:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 31b\n"
       "32:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 33f\n"
+      "cbz x20, 33f\n"
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 33f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 33f\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "33:"  // Store to output array: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 35f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 35f\n"
       "34:"  // Store to output array: Accumulator row 2 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 34b\n"
       "35:"  // Store to output array: Accumulator row 2 oddments
-      "cbz x19, 36f\n"
+      "cbz x20, 36f\n"
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 36f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 36f\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "36:"  // Store to output array: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 40f\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 38f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 38f\n"
       "37:"  // Store to output array: Accumulator row 3 loop
       ".inst 0xc0860474  // mova { z20.s-z23.s }, za3h.s[x12]\n"
       ".inst 0xc1b8cb34  // fclamp { z20.s-z23.s }, z25.s, z24.s\n"
-      "st1w { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1w { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1w { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "add x12, x12, #0x4\n"
-      "st1w { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 37b\n"
       "38:"  // Store to output array: Accumulator row 3 oddments
-      "cbz x19, 39f\n"
+      "cbz x20, 39f\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1b8cb30  // fclamp { z16.s-z19.s }, z25.s, z24.s\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 39f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 39f\n"
-      "st1w { z18.s }, p0, [x25]\n"
+      "st1w { z18.s }, p0, [x26]\n"
       "39:"  // Store to output array: Accumulator row 3 oddments: End
       "40:"  // Store to output array: End
-      "tbz x15, #0, 42f\n"
+      "tbz x16, #0, 42f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "41:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840682  // mova za2h.s[x12], { z20.s-z23.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 41b\n"
       "42:"  // End block
-      "incw x9\n"
-      "cmp x9, x28\n"
+      "incw x10\n"
+      "cmp x10, x9\n"
       "blt 3b\n"
-      "incw x10, ALL, MUL #4\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "incw x11, ALL, MUL #4\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_KernelArgs_max] "I" (offsetof(KernelArgs, max)), [offsetof_KernelArgs_min] "I" (offsetof(KernelArgs, min)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp
index 62170c4945..929af04032 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x13, [%x[args], %[offsetof_flags]]\n"
+      "ldr x14, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x11, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x10, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x13, #0, 2f\n"
+      "tbz x14, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa041c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840781  // mova za1h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa042c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840782  // mova za2h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa043c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      "addvl x13, x13, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w9, [%x[args], %[offsetof_M]]\n"
+      "ldr w10, [%x[args], %[offsetof_M]]\n"
+      "mov x9, #0x0\n"
       "mov x28, #0x0\n"
-      "mov x27, #0x0\n"
-      "ldr w26, [%x[args], %[offsetof_N]]\n"
-      "ldr x25, [%x[args], %[offsetof_A]]\n"
+      "ldr w27, [%x[args], %[offsetof_N]]\n"
+      "ldr x26, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x24, x25\n"
-      ".inst 0x25ba6770  // whilelt pn8.s, x27, x26, VLx4\n"
-      "tbnz x13, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x25, x26\n"
+      ".inst 0x25bb6790  // whilelt pn8.s, x28, x27, VLx4\n"
+      "tbnz x14, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa01bc279  // ldnt1w { z24.s-z27.s }, p8/Z, [x19, x27, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa01cc299  // ldnt1w { z24.s-z27.s }, p8/Z, [x20, x28, LSL #2]\n"
       ".inst 0xc0902700  // addha za0.s, p1/M, p1/M, z24.s\n"
       ".inst 0xc0902721  // addha za1.s, p1/M, p1/M, z25.s\n"
       ".inst 0xc0902742  // addha za2.s, p1/M, p1/M, z26.s\n"
       ".inst 0xc0902763  // addha za3.s, p1/M, p1/M, z27.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x27\n"
       "mov x20, x28\n"
-      "incw x19, ALL, MUL #4\n"
-      "incw x20\n"
-      "cmp x19, x26\n"
-      "csel x20, x28, x20, LT\n"
-      "mov x19, x13\n"
-      "bfm x13, XZR, #0x0, #0x0  // bfc x13, #0x0, #0x1\n"
-      "cmp x20, x9\n"
-      "csel x13, x19, x13, LT\n"
+      "mov x21, x9\n"
+      "incw x20, ALL, MUL #4\n"
+      "incw x21\n"
+      "cmp x20, x27\n"
+      "csel x21, x9, x21, LT\n"
+      "mov x20, x14\n"
+      "bfm x14, XZR, #0x0, #0x0  // bfc x14, #0x0, #0x1\n"
+      "cmp x21, x10\n"
+      "csel x14, x20, x14, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x27, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      "ld1b { z10.b }, p1/Z, [x24]\n"
-      ".inst 0xa04086dd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
-      "ld1b { z16.b }, p1/Z, [x24, #1, MUL VL]\n"
-      ".inst 0xa04186cd  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      "ld1b { z21.b }, p1/Z, [x24, #2, MUL VL]\n"
-      ".inst 0xa04286d9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
-      "ld1b { z19.b }, p1/Z, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
-      ".inst 0xa04386c1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x28, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
+      ".inst 0xa04086fd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
+      "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n"
+      ".inst 0xa04186ed  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n"
+      ".inst 0xa04286f9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
+      ".inst 0xa04386e1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa09c2540  // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa09d2541  // smopa za1.s, p1/M, p1/M, z10.b, z29.b\n"
       ".inst 0xa09e2542  // smopa za2.s, p1/M, p1/M, z10.b, z30.b\n"
       ".inst 0xa09f2543  // smopa za3.s, p1/M, p1/M, z10.b, z31.b\n"
-      "ld1b { z10.b }, p1/Z, [x24]\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
       ".inst 0xa08c2600  // smopa za0.s, p1/M, p1/M, z16.b, z12.b\n"
-      ".inst 0xa04086dd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa04086fd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa08d2601  // smopa za1.s, p1/M, p1/M, z16.b, z13.b\n"
       ".inst 0xa08e2602  // smopa za2.s, p1/M, p1/M, z16.b, z14.b\n"
       ".inst 0xa08f2603  // smopa za3.s, p1/M, p1/M, z16.b, z15.b\n"
-      "ld1b { z16.b }, p1/Z, [x24, #1, MUL VL]\n"
+      "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n"
       ".inst 0xa09826a0  // smopa za0.s, p1/M, p1/M, z21.b, z24.b\n"
-      ".inst 0xa04186cd  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa04186ed  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa09926a1  // smopa za1.s, p1/M, p1/M, z21.b, z25.b\n"
       ".inst 0xa09a26a2  // smopa za2.s, p1/M, p1/M, z21.b, z26.b\n"
       ".inst 0xa09b26a3  // smopa za3.s, p1/M, p1/M, z21.b, z27.b\n"
-      "ld1b { z21.b }, p1/Z, [x24, #2, MUL VL]\n"
-      ".inst 0xa04286d9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
+      "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n"
+      ".inst 0xa04286f9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
       ".inst 0xa0802660  // smopa za0.s, p1/M, p1/M, z19.b, z0.b\n"
       ".inst 0xa0812661  // smopa za1.s, p1/M, p1/M, z19.b, z1.b\n"
       ".inst 0xa0822662  // smopa za2.s, p1/M, p1/M, z19.b, z2.b\n"
       ".inst 0xa0832663  // smopa za3.s, p1/M, p1/M, z19.b, z3.b\n"
-      "ld1b { z19.b }, p1/Z, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
-      ".inst 0xa04386c1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
+      ".inst 0xa04386e1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa09c2540  // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
@@ -210,76 +210,76 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
       ".inst 0xa0822662  // smopa za2.s, p1/M, p1/M, z19.b, z2.b\n"
       ".inst 0xa0832663  // smopa za3.s, p1/M, p1/M, z19.b, z3.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      "ld1b { z10.b }, p1/Z, [x24]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x24, x24, #1\n"
-      ".inst 0xa04086dc  // ld1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #4\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x25, x25, #1\n"
+      ".inst 0xa04086fc  // ld1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #4\n"
       ".inst 0xa09c2540  // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
       ".inst 0xa09d2541  // smopa za1.s, p1/M, p1/M, z10.b, z29.b\n"
       ".inst 0xa09e2542  // smopa za2.s, p1/M, p1/M, z10.b, z30.b\n"
       ".inst 0xa09f2543  // smopa za3.s, p1/M, p1/M, z10.b, z31.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "ld1w { z14.s }, p1/Z, [x24]\n"
-      "addvl x24, x24, #1\n"
+      "ld1w { z14.s }, p1/Z, [x25]\n"
+      "addvl x25, x25, #1\n"
       ".inst 0xc09125c0  // addva za0.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c1  // addva za1.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c2  // addva za2.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c3  // addva za3.s, p1/M, p1/M, z14.s\n"
-      "tbz x13, #1, 14f\n"
-      "tbz x13, #0, 12f\n"
+      "tbz x14, #1, 14f\n"
+      "tbz x14, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c578  // ld1w { z24.s-z27.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5b8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc0840700  // mova za0h.s[x12], { z24.s-z27.s }\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa041c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
       ".inst 0xc0860458  // mova { z24.s-z27.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa042c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c55c  // st1w { z28.s-z31.s }, pn9.b, [x10]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c57c  // st1w { z28.s-z31.s }, pn9.b, [x11]\n"
+      "addvl x13, x13, #16\n"
+      ".inst 0xa061c568  // st1w { z8.s-z11.s }, pn9.b, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa062c578  // st1w { z24.s-z27.s }, pn9.b, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa063c56c  // st1w { z12.s-z15.s }, pn9.b, [x11, #0xc, MUL VL]\n"
       "addvl x11, x11, #16\n"
-      ".inst 0xa061c548  // st1w { z8.s-z11.s }, pn9.b, [x10, #0x4, MUL VL]\n"
-      ".inst 0xa062c558  // st1w { z24.s-z27.s }, pn9.b, [x10, #0x8, MUL VL]\n"
-      ".inst 0xa063c54c  // st1w { z12.s-z15.s }, pn9.b, [x10, #0xc, MUL VL]\n"
-      "addvl x10, x10, #16\n"
       "blt 11b\n"
       "b 21f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc0860420  // mova { z0.s-z3.s }, za1h.s[x12]\n"
-      ".inst 0xa060c55c  // st1w { z28.s-z31.s }, pn9.b, [x10]\n"
+      ".inst 0xa060c57c  // st1w { z28.s-z31.s }, pn9.b, [x11]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa061c540  // st1w { z0.s-z3.s }, pn9.b, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa061c560  // st1w { z0.s-z3.s }, pn9.b, [x11, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c548  // st1w { z8.s-z11.s }, pn9.b, [x10, #0x8, MUL VL]\n"
-      ".inst 0xa063c550  // st1w { z16.s-z19.s }, pn9.b, [x10, #0xc, MUL VL]\n"
-      "addvl x10, x10, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c568  // st1w { z8.s-z11.s }, pn9.b, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa063c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n"
+      "addvl x11, x11, #16\n"
       "blt 13b\n"
       "b 21f\n"
       "14:"  // Store to output array
-      "ldr x23, [%x[args], %[offsetof_C]]\n"
-      "add x23, x23, x27\n"  // C += n
-      "sub x22, x9, x28\n"
+      "ldr x24, [%x[args], %[offsetof_C]]\n"
+      "add x24, x24, x28\n"  // C += n
+      "sub x23, x10, x9\n"
       "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x21, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x23, x28, x21, x23\n"  // C += m * ldc
+      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x24, x9, x22, x24\n"  // C += m * ldc
       "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
@@ -290,24 +290,24 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
       "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x13, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x27\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa040c26c  // ld1w { z12.s-z15.s }, p8/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa040c264  // ld1w { z4.s-z7.s }, p8/Z, [x19]\n"
+      "tbz x14, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x28\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa040c28c  // ld1w { z12.s-z15.s }, p8/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa040c284  // ld1w { z4.s-z7.s }, p8/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x19\n"
-      "whilelt p0.b, x27, x26\n"
-      "cmp x22, x19\n"
-      "csel x19, x22, x19, LT\n"
-      "lsr x20, x19, #0x1\n"
+      "cntw x20\n"
+      "whilelt p0.b, x28, x27\n"
+      "cmp x23, x20\n"
+      "csel x20, x23, x20, LT\n"
+      "lsr x21, x20, #0x1\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x1\n"
-      "cbz x20, 17f\n"
+      "and x20, x20, #0x1\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086001a  // mova { z26.s-z27.s }, za0h.s[x12, 0:1]\n"
       ".inst 0xc086005c  // mova { z28.s-z29.s }, za1h.s[x12, 0:1]\n"
@@ -317,7 +317,7 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
       ".inst 0xc1ada41c  // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n"
       ".inst 0xc1aea416  // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n"
       "add x12, x12, #0x2\n"
-      "cmp x12, x20, LSL #1\n"
+      "cmp x12, x21, LSL #1\n"
       ".inst 0xc1afa410  // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n"
       ".inst 0xc1a4a23a  // srshl { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n"
       ".inst 0xc1a5a23c  // srshl { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n"
@@ -336,14 +336,14 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
       "uzp1 z18.b, z27.b, z29.b\n"
       "uzp1 z17.b, z23.b, z17.b\n"
       "uzp1 z16.b, z19.b, z16.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
-      "add x23, x23, x21\n"
+      "st1b { z16.b }, p0, [x24]\n"
+      "add x24, x24, x22\n"
       "uzp1 z16.b, z18.b, z17.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
-      "add x23, x23, x21\n"
+      "st1b { z16.b }, p0, [x24]\n"
+      "add x24, x24, x22\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc0860002  // mova { z2.s-z3.s }, za0h.s[x12, 0:1]\n"
       ".inst 0xc0860058  // mova { z24.s-z25.s }, za1h.s[x12, 0:1]\n"
       ".inst 0xc1aca402  // sqdmulh { z2.s-z3.s }, { z2.s-z3.s }, z12.s\n"
@@ -367,38 +367,38 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8
       ".inst 0xc1b4c6aa  // sclamp { z10.s-z11.s }, z21.s, z20.s\n"
       "uzp1 z16.b, z16.b, z10.b\n"
       "uzp1 z16.b, z23.b, z16.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
+      "st1b { z16.b }, p0, [x24]\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
       "19:"  // Store to output array: End
-      "tbz x13, #0, 21f\n"
+      "tbz x14, #0, 21f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "20:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      "addvl x13, x13, #16\n"
       "blt 20b\n"
       "21:"  // End block
-      "incw x27, ALL, MUL #4\n"
-      "cmp x27, x26\n"
+      "incw x28, ALL, MUL #4\n"
+      "cmp x28, x27\n"
       "blt 3b\n"
-      "incw x28\n"
-      "cmp x28, x9\n"
-      "mov x27, #0x0\n"
-      "mov x25, x24\n"
+      "incw x9\n"
+      "cmp x9, x10\n"
+      "mov x28, #0x0\n"
+      "mov x26, x25\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp
index e565699af5..0b642818e2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840400  // mova za0h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa041c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      ".inst 0x25bc4530  // whilelt pn8.s, x9, x28, VLx2\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      ".inst 0x25a94550  // whilelt pn8.s, x10, x9, VLx2\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa0094275  // ldnt1w { z20.s-z21.s }, p8/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa00a4295  // ldnt1w { z20.s-z21.s }, p8/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc0902680  // addha za0.s, p1/M, p1/M, z20.s\n"
       ".inst 0xc09026a1  // addha za1.s, p1/M, p1/M, z21.s\n"
       ".inst 0xc0902682  // addha za2.s, p1/M, p1/M, z20.s\n"
       ".inst 0xc09026a3  // addha za3.s, p1/M, p1/M, z21.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19, ALL, MUL #2\n"
+      "mov x21, x11\n"
       "incw x20, ALL, MUL #2\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "incw x21, ALL, MUL #2\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
-      ".inst 0xa04006d1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
-      ".inst 0xa041074e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
-      ".inst 0xa04106c9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
-      ".inst 0xa0420740  // ld1b { z0.b-z1.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa14206dc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306ca  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
+      ".inst 0xa04006f1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
+      ".inst 0xa041076e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
+      ".inst 0xa04106e9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
+      ".inst 0xa0420760  // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa14206fc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306ea  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa09027c0  // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa09127c1  // smopa za1.s, p1/M, p1/M, z30.b, z17.b\n"
       ".inst 0xa09027e2  // smopa za2.s, p1/M, p1/M, z31.b, z16.b\n"
       ".inst 0xa09127e3  // smopa za3.s, p1/M, p1/M, z31.b, z17.b\n"
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
       ".inst 0xa08825c0  // smopa za0.s, p1/M, p1/M, z14.b, z8.b\n"
-      ".inst 0xa04006d1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa04006f1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa08925c1  // smopa za1.s, p1/M, p1/M, z14.b, z9.b\n"
       ".inst 0xa08825e2  // smopa za2.s, p1/M, p1/M, z15.b, z8.b\n"
       ".inst 0xa08925e3  // smopa za3.s, p1/M, p1/M, z15.b, z9.b\n"
-      ".inst 0xa041074e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
+      ".inst 0xa041076e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
       ".inst 0xa0942400  // smopa za0.s, p1/M, p1/M, z0.b, z20.b\n"
-      ".inst 0xa04106c9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
+      ".inst 0xa04106e9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
       ".inst 0xa09c2401  // smopa za1.s, p1/M, p1/M, z0.b, z28.b\n"
       ".inst 0xa0942422  // smopa za2.s, p1/M, p1/M, z1.b, z20.b\n"
       ".inst 0xa09c2423  // smopa za3.s, p1/M, p1/M, z1.b, z28.b\n"
-      ".inst 0xa0420740  // ld1b { z0.b-z1.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa14206dc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa0420760  // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa14206fc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa0822480  // smopa za0.s, p1/M, p1/M, z4.b, z2.b\n"
       ".inst 0xa08a2481  // smopa za1.s, p1/M, p1/M, z4.b, z10.b\n"
       ".inst 0xa08224a2  // smopa za2.s, p1/M, p1/M, z5.b, z2.b\n"
       ".inst 0xa08a24a3  // smopa za3.s, p1/M, p1/M, z5.b, z10.b\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306ca  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306ea  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa09027c0  // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
@@ -210,107 +210,107 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
       ".inst 0xa08224a2  // smopa za2.s, p1/M, p1/M, z5.b, z2.b\n"
       ".inst 0xa08a24a3  // smopa za3.s, p1/M, p1/M, z5.b, z10.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #2\n"
-      ".inst 0xa04006d0  // ld1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #2\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #2\n"
+      ".inst 0xa04006f0  // ld1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #2\n"
       ".inst 0xa09027c0  // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
       ".inst 0xa09127c1  // smopa za1.s, p1/M, p1/M, z30.b, z17.b\n"
       ".inst 0xa09027e2  // smopa za2.s, p1/M, p1/M, z31.b, z16.b\n"
       ".inst 0xa09127e3  // smopa za3.s, p1/M, p1/M, z31.b, z17.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      ".inst 0xa040474e  // ld1w { z14.s-z15.s }, pn9.b/Z, [x26]\n"
-      "addvl x26, x26, #2\n"
+      ".inst 0xa040476e  // ld1w { z14.s-z15.s }, pn9.b/Z, [x27]\n"
+      "addvl x27, x27, #2\n"
       ".inst 0xc09125c0  // addva za0.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c1  // addva za1.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125e2  // addva za2.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09125e3  // addva za3.s, p1/M, p1/M, z15.s\n"
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
       ".inst 0xc0860434  // mova { z20.s-z23.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840702  // mova za2h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa043c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c5d4  // st1w { z20.s-z23.s }, pn9.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c5dc  // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 24f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      ".inst 0xa060c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13]\n"
+      ".inst 0xa060c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa061c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c5c4  // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 24f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9\n"  // C += n
-      "sub x24, x11, x10\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10\n"  // C += n
+      "sub x25, x13, x11\n"
       "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
       "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z11.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z25.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z24.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x15, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x9\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa0404262  // ld1w { z2.s-z3.s }, p8/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa0404260  // ld1w { z0.s-z1.s }, p8/Z, [x19]\n"
+      "tbz x16, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x10\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa0404282  // ld1w { z2.s-z3.s }, p8/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa0404280  // ld1w { z0.s-z1.s }, p8/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x22\n"
-      "whilelt p0.h, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cntw x23\n"
+      "whilelt p0.h, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 17f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
       ".inst 0xc1a2ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n"
       ".inst 0xc1a3ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z3.s\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a0aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n"
       ".inst 0xc1a1aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n"
       ".inst 0xc1abab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n"
@@ -318,25 +318,25 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
       ".inst 0xc1b8cf2c  // sclamp { z12.s-z15.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf3c  // sclamp { z28.s-z31.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z12.h, z28.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z13.h, z29.h\n"
       "uzp1 z17.h, z14.h, z30.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z15.h, z31.h\n"
-      "st1b { z17.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z17.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
       ".inst 0xc1a2ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n"
       ".inst 0xc1a3ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a0aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n"
       ".inst 0xc1a1aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n"
       ".inst 0xc1abab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n"
@@ -344,34 +344,34 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
       ".inst 0xc1b8cf3c  // sclamp { z28.s-z31.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf2c  // sclamp { z12.s-z15.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z28.h, z12.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       "uzp1 z16.h, z29.h, z13.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
       "uzp1 z16.h, z30.h, z14.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 22f\n"
-      "whilelt p0.h, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "whilelt p0.h, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
       ".inst 0xc1a2ac04  // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n"
       ".inst 0xc1a3ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a0aa24  // srshl { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n"
       ".inst 0xc1a1aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n"
       ".inst 0xc1abab04  // add { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n"
@@ -379,25 +379,25 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
       ".inst 0xc1b8cf24  // sclamp { z4.s-z7.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf30  // sclamp { z16.s-z19.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z4.h, z16.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z5.h, z17.h\n"
       "uzp1 z17.h, z6.h, z18.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z7.h, z19.h\n"
-      "st1b { z17.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z17.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
       ".inst 0xc1a2ac14  // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z2.s\n"
       ".inst 0xc1a3ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a0aa34  // srshl { z20.s-z23.s }, { z20.s-z23.s }, z0.s\n"
       ".inst 0xc1a1aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n"
       ".inst 0xc1abab14  // add { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n"
@@ -405,47 +405,47 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8
       ".inst 0xc1b8cf34  // sclamp { z20.s-z23.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf30  // sclamp { z16.s-z19.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z20.h, z16.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       "uzp1 z16.h, z21.h, z17.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
       "uzp1 z16.h, z22.h, z18.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
+      "st1b { z16.h }, p0, [x26]\n"
       "21:"  // Store to output array: Accumulator row 1 oddments: End
       "22:"  // Store to output array: End
-      "tbz x15, #0, 24f\n"
+      "tbz x16, #0, 24f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "23:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 23b\n"
       "24:"  // End block
-      "incw x9, ALL, MUL #2\n"
-      "cmp x9, x28\n"
-      "blt 3b\n"
       "incw x10, ALL, MUL #2\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "cmp x10, x9\n"
+      "blt 3b\n"
+      "incw x11, ALL, MUL #2\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp
index a738a10418..0d0e3da224 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c1dc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1fc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa041c1cc  // ld1w { z12.s-z15.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1ec  // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840682  // mova za2h.s[x12], { z20.s-z23.s }\n"
-      ".inst 0xa043c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840703  // mova za3h.s[x12], { z24.s-z27.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      "whilelt p0.s, x9, x28\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      "whilelt p0.s, x10, x9\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      "ldnt1w { z15.s }, p0/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc09025e0  // addha za0.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e1  // addha za1.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e2  // addha za2.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e3  // addha za3.s, p1/M, p1/M, z15.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19\n"
-      "incw x20, ALL, MUL #4\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "mov x21, x11\n"
+      "incw x20\n"
+      "incw x21, ALL, MUL #4\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
-      "ldnt1b { z0.b }, p1/Z, [x22]\n"
-      ".inst 0xa1418353  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
-      "ldnt1b { z9.b }, p1/Z, [x22, #1, MUL VL]\n"
-      ".inst 0xa1428350  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z21.b }, p1/Z, [x22, #2, MUL VL]\n"
-      ".inst 0xa1438342  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
+      "ldnt1b { z0.b }, p1/Z, [x23]\n"
+      ".inst 0xa1418373  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
+      "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n"
+      ".inst 0xa1428370  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n"
+      ".inst 0xa1438362  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa0802640  // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa08026c1  // smopa za1.s, p1/M, p1/M, z22.b, z0.b\n"
       ".inst 0xa0802742  // smopa za2.s, p1/M, p1/M, z26.b, z0.b\n"
       ".inst 0xa08027c3  // smopa za3.s, p1/M, p1/M, z30.b, z0.b\n"
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
       ".inst 0xa0892660  // smopa za0.s, p1/M, p1/M, z19.b, z9.b\n"
-      "ldnt1b { z0.b }, p1/Z, [x22]\n"
+      "ldnt1b { z0.b }, p1/Z, [x23]\n"
       ".inst 0xa08926e1  // smopa za1.s, p1/M, p1/M, z23.b, z9.b\n"
       ".inst 0xa0892762  // smopa za2.s, p1/M, p1/M, z27.b, z9.b\n"
       ".inst 0xa08927e3  // smopa za3.s, p1/M, p1/M, z31.b, z9.b\n"
-      ".inst 0xa1418353  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
+      ".inst 0xa1418373  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
       ".inst 0xa0952600  // smopa za0.s, p1/M, p1/M, z16.b, z21.b\n"
-      "ldnt1b { z9.b }, p1/Z, [x22, #1, MUL VL]\n"
+      "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n"
       ".inst 0xa0952681  // smopa za1.s, p1/M, p1/M, z20.b, z21.b\n"
       ".inst 0xa0952702  // smopa za2.s, p1/M, p1/M, z24.b, z21.b\n"
       ".inst 0xa0952783  // smopa za3.s, p1/M, p1/M, z28.b, z21.b\n"
-      ".inst 0xa1428350  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z21.b }, p1/Z, [x22, #2, MUL VL]\n"
+      ".inst 0xa1428370  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n"
       ".inst 0xa08c2440  // smopa za0.s, p1/M, p1/M, z2.b, z12.b\n"
       ".inst 0xa08c24c1  // smopa za1.s, p1/M, p1/M, z6.b, z12.b\n"
       ".inst 0xa08c2542  // smopa za2.s, p1/M, p1/M, z10.b, z12.b\n"
       ".inst 0xa08c25c3  // smopa za3.s, p1/M, p1/M, z14.b, z12.b\n"
-      ".inst 0xa1438342  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      ".inst 0xa1438362  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa0802640  // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
@@ -210,294 +210,294 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8
       ".inst 0xa08c2542  // smopa za2.s, p1/M, p1/M, z10.b, z12.b\n"
       ".inst 0xa08c25c3  // smopa za3.s, p1/M, p1/M, z14.b, z12.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #4\n"
-      "ld1b { z0.b }, p1/Z, [x22]\n"
-      "addvl x22, x22, #1\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #4\n"
+      "ld1b { z0.b }, p1/Z, [x23]\n"
+      "addvl x23, x23, #1\n"
       ".inst 0xa0802640  // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
       ".inst 0xa08026c1  // smopa za1.s, p1/M, p1/M, z22.b, z0.b\n"
       ".inst 0xa0802742  // smopa za2.s, p1/M, p1/M, z26.b, z0.b\n"
       ".inst 0xa08027c3  // smopa za3.s, p1/M, p1/M, z30.b, z0.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      ".inst 0xa040c340  // ld1w { z0.s-z3.s }, pn8.b/Z, [x26]\n"
-      "addvl x26, x26, #4\n"
+      ".inst 0xa040c360  // ld1w { z0.s-z3.s }, pn8.b/Z, [x27]\n"
+      "addvl x27, x27, #4\n"
       ".inst 0xc0912400  // addva za0.s, p1/M, p1/M, z0.s\n"
       ".inst 0xc0912421  // addva za1.s, p1/M, p1/M, z1.s\n"
       ".inst 0xc0912442  // addva za2.s, p1/M, p1/M, z2.s\n"
       ".inst 0xc0912463  // addva za3.s, p1/M, p1/M, z3.s\n"
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0840680  // mova za0h.s[x12], { z20.s-z23.s }\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa041c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840481  // mova za1h.s[x12], { z4.s-z7.s }\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc086047c  // mova { z28.s-z31.s }, za3h.s[x12]\n"
-      ".inst 0xa042c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840683  // mova za3h.s[x12], { z20.s-z23.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c1b0  // st1w { z16.s-z19.s }, pn8.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c1d0  // st1w { z16.s-z19.s }, pn8.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1dc  // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1bc  // st1w { z28.s-z31.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 30f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa060c1b0  // st1w { z16.s-z19.s }, pn8.b, [x13]\n"
+      ".inst 0xa060c1d0  // st1w { z16.s-z19.s }, pn8.b, [x14]\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
-      ".inst 0xa061c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c1b4  // st1w { z20.s-z23.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c1d4  // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 30f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9\n"  // C += n
-      "sub x24, x11, x10\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10\n"  // C += n
+      "sub x25, x13, x11\n"
       "ld1rw { z8.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
       "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x15, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x9\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      "ld1w { z8.s }, p0/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      "ld1w { z7.s }, p0/Z, [x19]\n"
+      "tbz x16, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x10\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      "ld1w { z8.s }, p0/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      "ld1w { z7.s }, p0/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x22\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cntw x23\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 17f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc1a8ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
       ".inst 0xc1a4ccac  // sclamp { z12.s-z15.s }, z5.s, z4.s\n"
-      "st1b { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z15.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z15.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc1a8ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n"
       ".inst 0xc1a6ab10  // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n"
       ".inst 0xc1a4ccb0  // sclamp { z16.s-z19.s }, z5.s, z4.s\n"
-      "st1b { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "st1b { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
       ".inst 0xc1a8ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab10  // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n"
       ".inst 0xc1a4ccb0  // sclamp { z16.s-z19.s }, z5.s, z4.s\n"
-      "st1b { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
       ".inst 0xc1a8ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1a4ccbc  // sclamp { z28.s-z31.s }, z5.s, z4.s\n"
-      "st1b { z28.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z28.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z29.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z29.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "st1b { z30.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z30.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "21:"  // Store to output array: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 23f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 23f\n"
       "22:"  // Store to output array: Accumulator row 2 loop
       ".inst 0xc0860458  // mova { z24.s-z27.s }, za2h.s[x12]\n"
       ".inst 0xc1a8ac18  // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa38  // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab18  // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n"
       ".inst 0xc1a4ccb8  // sclamp { z24.s-z27.s }, z5.s, z4.s\n"
-      "st1b { z24.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z25.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z26.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z27.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z24.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z25.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z26.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z27.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 22b\n"
       "23:"  // Store to output array: Accumulator row 2 oddments
-      "cbz x19, 24f\n"
+      "cbz x20, 24f\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc1a8ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
       ".inst 0xc1a4ccac  // sclamp { z12.s-z15.s }, z5.s, z4.s\n"
-      "st1b { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "st1b { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "24:"  // Store to output array: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 26f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 26f\n"
       "25:"  // Store to output array: Accumulator row 3 loop
       ".inst 0xc0860474  // mova { z20.s-z23.s }, za3h.s[x12]\n"
       ".inst 0xc1a8ac14  // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa34  // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab14  // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n"
       ".inst 0xc1a4ccb4  // sclamp { z20.s-z23.s }, z5.s, z4.s\n"
-      "st1b { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 25b\n"
       "26:"  // Store to output array: Accumulator row 3 oddments
-      "cbz x19, 27f\n"
+      "cbz x20, 27f\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
       ".inst 0xc1a8ac00  // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z7.s\n"
       ".inst 0xc1a6ab00  // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n"
       ".inst 0xc1a4cca0  // sclamp { z0.s-z3.s }, z5.s, z4.s\n"
-      "st1b { z0.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z0.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z1.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z1.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "st1b { z2.s }, p0, [x25]\n"
+      "st1b { z2.s }, p0, [x26]\n"
       "27:"  // Store to output array: Accumulator row 3 oddments: End
       "28:"  // Store to output array: End
-      "tbz x15, #0, 30f\n"
+      "tbz x16, #0, 30f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "29:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840480  // mova za0h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa041c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 29b\n"
       "30:"  // End block
-      "incw x9\n"
-      "cmp x9, x28\n"
+      "incw x10\n"
+      "cmp x10, x9\n"
       "blt 3b\n"
-      "incw x10, ALL, MUL #4\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "incw x11, ALL, MUL #4\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp
index 7ddd7c2e09..12e714a471 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -84,107 +84,107 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in
   KernelArgs args(A, B, C, ldc, M, N, K, bias, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x11, [%x[args], %[offsetof_flags]]\n"
+      "ldr x13, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x11, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x10, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x9, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x11, #0, 2f\n"
+      "tbz x13, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c544  // ld1w { z4.s-z7.s }, pn9.b/Z, [x10]\n"
+      ".inst 0xa040c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11]\n"
       ".inst 0xc0840480  // mova za0h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa041c55c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa041c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
       ".inst 0xc0840781  // mova za1h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa042c550  // ld1w { z16.s-z19.s }, pn9.b/Z, [x10, #0x8, MUL VL]\n"
+      ".inst 0xa042c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c540  // ld1w { z0.s-z3.s }, pn9.b/Z, [x10, #0xc, MUL VL]\n"
+      ".inst 0xa043c560  // ld1w { z0.s-z3.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
       ".inst 0xc0840403  // mova za3h.s[x12], { z0.s-z3.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x10, x10, #16\n"
+      "cmp x12, x20\n"
+      "addvl x11, x11, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w28, [%x[args], %[offsetof_M]]\n"
+      "ldr w9, [%x[args], %[offsetof_M]]\n"
+      "mov x28, #0x0\n"
       "mov x27, #0x0\n"
-      "mov x26, #0x0\n"
-      "ldr w25, [%x[args], %[offsetof_N]]\n"
-      "ldr x24, [%x[args], %[offsetof_A]]\n"
+      "ldr w26, [%x[args], %[offsetof_N]]\n"
+      "ldr x25, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x23, x24\n"
-      ".inst 0x25b96750  // whilelt pn8.s, x26, x25, VLx4\n"
-      "tbnz x11, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x24, x25\n"
+      ".inst 0x25ba6770  // whilelt pn8.s, x27, x26, VLx4\n"
+      "tbnz x13, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa11ac26a  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, p8/Z, [x19, x26, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa11bc28a  // ldnt1w { z2.s, z6.s, z10.s, z14.s }, p8/Z, [x20, x27, LSL #2]\n"
       ".inst 0xc0900040  // addha za0.s, p0/M, p0/M, z2.s\n"
       ".inst 0xc09000c1  // addha za1.s, p0/M, p0/M, z6.s\n"
       ".inst 0xc0900142  // addha za2.s, p0/M, p0/M, z10.s\n"
       ".inst 0xc09001c3  // addha za3.s, p0/M, p0/M, z14.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x26\n"
       "mov x20, x27\n"
-      "incw x19, ALL, MUL #4\n"
-      "incw x20\n"
-      "cmp x19, x25\n"
-      "csel x20, x27, x20, LT\n"
-      "mov x19, x11\n"
-      "bfm x11, XZR, #0x0, #0x0  // bfc x11, #0x0, #0x1\n"
-      "cmp x20, x28\n"
-      "csel x11, x19, x11, LT\n"
+      "mov x21, x28\n"
+      "incw x20, ALL, MUL #4\n"
+      "incw x21\n"
+      "cmp x20, x26\n"
+      "csel x21, x28, x21, LT\n"
+      "mov x20, x13\n"
+      "bfm x13, XZR, #0x0, #0x0  // bfc x13, #0x0, #0x1\n"
+      "cmp x21, x9\n"
+      "csel x13, x20, x13, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x26, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      "ld1b { z20.b }, p0/Z, [x23]\n"
-      ".inst 0xa14086c9  // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x22]\n"
-      "ld1b { z10.b }, p0/Z, [x23, #1, MUL VL]\n"
-      ".inst 0xa14186da  // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      "ld1b { z16.b }, p0/Z, [x23, #2, MUL VL]\n"
-      ".inst 0xa14286cb  // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
-      "ld1b { z25.b }, p0/Z, [x23, #3, MUL VL]\n"
-      "addvl x23, x23, #4\n"
-      ".inst 0xa14386c8  // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x27, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      "ld1b { z20.b }, p0/Z, [x24]\n"
+      ".inst 0xa14086e9  // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n"
+      "ld1b { z10.b }, p0/Z, [x24, #1, MUL VL]\n"
+      ".inst 0xa14186fa  // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      "ld1b { z16.b }, p0/Z, [x24, #2, MUL VL]\n"
+      ".inst 0xa14286eb  // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      "ld1b { z25.b }, p0/Z, [x24, #3, MUL VL]\n"
+      "addvl x24, x24, #4\n"
+      ".inst 0xa14386e8  // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa0810280  // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa0850281  // smopa za1.s, p0/M, p0/M, z20.b, z5.b\n"
       ".inst 0xa0890282  // smopa za2.s, p0/M, p0/M, z20.b, z9.b\n"
       ".inst 0xa08d0283  // smopa za3.s, p0/M, p0/M, z20.b, z13.b\n"
-      "ld1b { z20.b }, p0/Z, [x23]\n"
+      "ld1b { z20.b }, p0/Z, [x24]\n"
       ".inst 0xa0920140  // smopa za0.s, p0/M, p0/M, z10.b, z18.b\n"
-      ".inst 0xa14086c9  // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa14086e9  // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa0960141  // smopa za1.s, p0/M, p0/M, z10.b, z22.b\n"
       ".inst 0xa09a0142  // smopa za2.s, p0/M, p0/M, z10.b, z26.b\n"
       ".inst 0xa09e0143  // smopa za3.s, p0/M, p0/M, z10.b, z30.b\n"
-      "ld1b { z10.b }, p0/Z, [x23, #1, MUL VL]\n"
+      "ld1b { z10.b }, p0/Z, [x24, #1, MUL VL]\n"
       ".inst 0xa0830200  // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n"
-      ".inst 0xa14186da  // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa14186fa  // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa0870201  // smopa za1.s, p0/M, p0/M, z16.b, z7.b\n"
       ".inst 0xa08b0202  // smopa za2.s, p0/M, p0/M, z16.b, z11.b\n"
       ".inst 0xa08f0203  // smopa za3.s, p0/M, p0/M, z16.b, z15.b\n"
-      "ld1b { z16.b }, p0/Z, [x23, #2, MUL VL]\n"
-      ".inst 0xa14286cb  // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
+      "ld1b { z16.b }, p0/Z, [x24, #2, MUL VL]\n"
+      ".inst 0xa14286eb  // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
       ".inst 0xa0800320  // smopa za0.s, p0/M, p0/M, z25.b, z0.b\n"
       ".inst 0xa0840321  // smopa za1.s, p0/M, p0/M, z25.b, z4.b\n"
       ".inst 0xa0880322  // smopa za2.s, p0/M, p0/M, z25.b, z8.b\n"
       ".inst 0xa08c0323  // smopa za3.s, p0/M, p0/M, z25.b, z12.b\n"
-      "ld1b { z25.b }, p0/Z, [x23, #3, MUL VL]\n"
-      "addvl x23, x23, #4\n"
-      ".inst 0xa14386c8  // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ld1b { z25.b }, p0/Z, [x24, #3, MUL VL]\n"
+      "addvl x24, x24, #4\n"
+      ".inst 0xa14386e8  // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa0810280  // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n"
@@ -204,138 +204,138 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in
       ".inst 0xa0880322  // smopa za2.s, p0/M, p0/M, z25.b, z8.b\n"
       ".inst 0xa08c0323  // smopa za3.s, p0/M, p0/M, z25.b, z12.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      "ld1b { z20.b }, p0/Z, [x23]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x23, x23, #1\n"
-      ".inst 0xa14086c1  // ld1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #4\n"
+      "ld1b { z20.b }, p0/Z, [x24]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x24, x24, #1\n"
+      ".inst 0xa14086e1  // ld1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #4\n"
       ".inst 0xa0810280  // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n"
       ".inst 0xa0850281  // smopa za1.s, p0/M, p0/M, z20.b, z5.b\n"
       ".inst 0xa0890282  // smopa za2.s, p0/M, p0/M, z20.b, z9.b\n"
       ".inst 0xa08d0283  // smopa za3.s, p0/M, p0/M, z20.b, z13.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x11, #1, 14f\n"
-      "tbz x11, #0, 12f\n"
+      "tbz x13, #1, 14f\n"
+      "tbz x13, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c55c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x10]\n"
+      ".inst 0xa040c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11]\n"
       ".inst 0xc0860404  // mova { z4.s-z7.s }, za0h.s[x12]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
       ".inst 0xc0860434  // mova { z20.s-z23.s }, za1h.s[x12]\n"
-      ".inst 0xa041c540  // ld1w { z0.s-z3.s }, pn9.b/Z, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa041c560  // ld1w { z0.s-z3.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
       ".inst 0xc0840401  // mova za1h.s[x12], { z0.s-z3.s }\n"
       ".inst 0xc0860458  // mova { z24.s-z27.s }, za2h.s[x12]\n"
       ".inst 0xc086047c  // mova { z28.s-z31.s }, za3h.s[x12]\n"
-      ".inst 0xa042c550  // ld1w { z16.s-z19.s }, pn9.b/Z, [x10, #0x8, MUL VL]\n"
+      ".inst 0xa042c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c54c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x10, #0xc, MUL VL]\n"
+      ".inst 0xa043c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c524  // st1w { z4.s-z7.s }, pn9.b, [x9]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c544  // st1w { z4.s-z7.s }, pn9.b, [x10]\n"
+      "addvl x11, x11, #16\n"
+      ".inst 0xa061c554  // st1w { z20.s-z23.s }, pn9.b, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa062c558  // st1w { z24.s-z27.s }, pn9.b, [x10, #0x8, MUL VL]\n"
+      ".inst 0xa063c55c  // st1w { z28.s-z31.s }, pn9.b, [x10, #0xc, MUL VL]\n"
       "addvl x10, x10, #16\n"
-      ".inst 0xa061c534  // st1w { z20.s-z23.s }, pn9.b, [x9, #0x4, MUL VL]\n"
-      ".inst 0xa062c538  // st1w { z24.s-z27.s }, pn9.b, [x9, #0x8, MUL VL]\n"
-      ".inst 0xa063c53c  // st1w { z28.s-z31.s }, pn9.b, [x9, #0xc, MUL VL]\n"
-      "addvl x9, x9, #16\n"
       "blt 11b\n"
       "b 20f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860414  // mova { z20.s-z23.s }, za0h.s[x12]\n"
       ".inst 0xc0860420  // mova { z0.s-z3.s }, za1h.s[x12]\n"
-      ".inst 0xa060c534  // st1w { z20.s-z23.s }, pn9.b, [x9]\n"
+      ".inst 0xa060c554  // st1w { z20.s-z23.s }, pn9.b, [x10]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa061c520  // st1w { z0.s-z3.s }, pn9.b, [x9, #0x4, MUL VL]\n"
+      ".inst 0xa061c540  // st1w { z0.s-z3.s }, pn9.b, [x10, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c528  // st1w { z8.s-z11.s }, pn9.b, [x9, #0x8, MUL VL]\n"
-      ".inst 0xa063c52c  // st1w { z12.s-z15.s }, pn9.b, [x9, #0xc, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c548  // st1w { z8.s-z11.s }, pn9.b, [x10, #0x8, MUL VL]\n"
+      ".inst 0xa063c54c  // st1w { z12.s-z15.s }, pn9.b, [x10, #0xc, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "blt 13b\n"
       "b 20f\n"
       "14:"  // Store to output array
-      "ldr x22, [%x[args], %[offsetof_C]]\n"
-      "sub x20, x28, x27\n"
-      "cntw x19\n"
-      "ldr x21, [%x[args], %[offsetof_ldcb]]\n"
-      "cmp x20, x19\n"
-      "csel x19, x20, x19, LT\n"
-      "add x22, x22, x26, LSL #2\n"  // C += n
-      "lsr x20, x19, #0x2\n"
-      "madd x22, x27, x21, x22\n"  // C += m * ldc
+      "ldr x23, [%x[args], %[offsetof_C]]\n"
+      "sub x21, x9, x28\n"
+      "cntw x20\n"
+      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
+      "cmp x21, x20\n"
+      "csel x20, x21, x20, LT\n"
+      "add x23, x23, x27, LSL #2\n"  // C += n
+      "lsr x21, x20, #0x2\n"
+      "madd x23, x28, x22, x23\n"  // C += m * ldc
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c2c0  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
-      ".inst 0xa160c2c1  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
+      ".inst 0xa160c2e0  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
+      ".inst 0xa160c2e1  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa160c2c2  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa160c2c3  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
+      ".inst 0xa160c2e2  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa160c2e3  // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
       "blt 15b\n"
       "16:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa160c2c0  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
+      ".inst 0xa160c2e0  // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa160c2c1  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x22]\n"
-      "add x22, x22, x21\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa160c2e1  // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x23]\n"
+      "add x23, x23, x22\n"
       "beq 17f\n"
-      ".inst 0xa160c2c2  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x22]\n"
+      ".inst 0xa160c2e2  // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x23]\n"
       "17:"  // Store to output array: Accumulator row 0 oddments: End
       "18:"  // Store to output array: End
-      "tbz x11, #0, 20f\n"
+      "tbz x13, #0, 20f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "19:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c54c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x10]\n"
+      ".inst 0xa040c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa041c550  // ld1w { z16.s-z19.s }, pn9.b/Z, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa041c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c550  // ld1w { z16.s-z19.s }, pn9.b/Z, [x10, #0x8, MUL VL]\n"
+      ".inst 0xa042c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c544  // ld1w { z4.s-z7.s }, pn9.b/Z, [x10, #0xc, MUL VL]\n"
+      ".inst 0xa043c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x10, x10, #16\n"
+      "cmp x12, x20\n"
+      "addvl x11, x11, #16\n"
       "blt 19b\n"
       "20:"  // End block
-      "incw x26, ALL, MUL #4\n"
-      "cmp x26, x25\n"
+      "incw x27, ALL, MUL #4\n"
+      "cmp x27, x26\n"
       "blt 3b\n"
-      "incw x27\n"
-      "cmp x27, x28\n"
-      "mov x26, #0x0\n"
-      "mov x24, x23\n"
+      "incw x28\n"
+      "cmp x28, x9\n"
+      "mov x27, #0x0\n"
+      "mov x25, x24\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp
index 9ae18f0e6b..d7a7528211 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -84,107 +84,107 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in
   KernelArgs args(A, B, C, ldc, M, N, K, bias, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p0.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5c8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5e8  // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840500  // mova za0h.s[x12], { z8.s-z11.s }\n"
-      ".inst 0xa041c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa042c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840782  // mova za2h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa043c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840603  // mova za3h.s[x12], { z16.s-z19.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      ".inst 0x25bc4530  // whilelt pn8.s, x9, x28, VLx2\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      ".inst 0x25a94550  // whilelt pn8.s, x10, x9, VLx2\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa109427c  // ldnt1w { z20.s, z28.s }, p8/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa10a429c  // ldnt1w { z20.s, z28.s }, p8/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc0900280  // addha za0.s, p0/M, p0/M, z20.s\n"
       ".inst 0xc0900381  // addha za1.s, p0/M, p0/M, z28.s\n"
       ".inst 0xc0900282  // addha za2.s, p0/M, p0/M, z20.s\n"
       ".inst 0xc0900383  // addha za3.s, p0/M, p0/M, z28.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19, ALL, MUL #2\n"
+      "mov x21, x11\n"
       "incw x20, ALL, MUL #2\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "incw x21, ALL, MUL #2\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa1400756  // ld1b { z22.b, z30.b }, pn9.b/Z, [x26]\n"
-      ".inst 0xa14006d9  // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x22]\n"
-      ".inst 0xa1410750  // ld1b { z16.b, z24.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
-      ".inst 0xa14106cb  // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
-      ".inst 0xa0420748  // ld1b { z8.b-z9.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa04206d3  // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306dd  // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa1400776  // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n"
+      ".inst 0xa14006f9  // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n"
+      ".inst 0xa1410770  // ld1b { z16.b, z24.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
+      ".inst 0xa14106eb  // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
+      ".inst 0xa0420768  // ld1b { z8.b-z9.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04206f3  // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306fd  // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa09102c0  // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa09902c1  // smopa za1.s, p0/M, p0/M, z22.b, z25.b\n"
       ".inst 0xa09103c2  // smopa za2.s, p0/M, p0/M, z30.b, z17.b\n"
       ".inst 0xa09903c3  // smopa za3.s, p0/M, p0/M, z30.b, z25.b\n"
-      ".inst 0xa1400756  // ld1b { z22.b, z30.b }, pn9.b/Z, [x26]\n"
+      ".inst 0xa1400776  // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n"
       ".inst 0xa0830200  // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n"
-      ".inst 0xa14006d9  // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa14006f9  // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa08b0201  // smopa za1.s, p0/M, p0/M, z16.b, z11.b\n"
       ".inst 0xa0830302  // smopa za2.s, p0/M, p0/M, z24.b, z3.b\n"
       ".inst 0xa08b0303  // smopa za3.s, p0/M, p0/M, z24.b, z11.b\n"
-      ".inst 0xa1410750  // ld1b { z16.b, z24.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
+      ".inst 0xa1410770  // ld1b { z16.b, z24.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
       ".inst 0xa0920100  // smopa za0.s, p0/M, p0/M, z8.b, z18.b\n"
-      ".inst 0xa14106cb  // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
+      ".inst 0xa14106eb  // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
       ".inst 0xa0930101  // smopa za1.s, p0/M, p0/M, z8.b, z19.b\n"
       ".inst 0xa0920122  // smopa za2.s, p0/M, p0/M, z9.b, z18.b\n"
       ".inst 0xa0930123  // smopa za3.s, p0/M, p0/M, z9.b, z19.b\n"
-      ".inst 0xa0420748  // ld1b { z8.b-z9.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa04206d3  // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa0420768  // ld1b { z8.b-z9.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa04206f3  // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa0950080  // smopa za0.s, p0/M, p0/M, z4.b, z21.b\n"
       ".inst 0xa09d0081  // smopa za1.s, p0/M, p0/M, z4.b, z29.b\n"
       ".inst 0xa09500a2  // smopa za2.s, p0/M, p0/M, z5.b, z21.b\n"
       ".inst 0xa09d00a3  // smopa za3.s, p0/M, p0/M, z5.b, z29.b\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306dd  // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306fd  // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa09102c0  // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n"
@@ -204,171 +204,171 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in
       ".inst 0xa09500a2  // smopa za2.s, p0/M, p0/M, z5.b, z21.b\n"
       ".inst 0xa09d00a3  // smopa za3.s, p0/M, p0/M, z5.b, z29.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa1400756  // ld1b { z22.b, z30.b }, pn9.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #2\n"
-      ".inst 0xa14006d1  // ld1b { z17.b, z25.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #2\n"
+      ".inst 0xa1400776  // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #2\n"
+      ".inst 0xa14006f1  // ld1b { z17.b, z25.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #2\n"
       ".inst 0xa09102c0  // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n"
       ".inst 0xa09902c1  // smopa za1.s, p0/M, p0/M, z22.b, z25.b\n"
       ".inst 0xa09103c2  // smopa za2.s, p0/M, p0/M, z30.b, z17.b\n"
       ".inst 0xa09903c3  // smopa za3.s, p0/M, p0/M, z30.b, z25.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0860418  // mova { z24.s-z27.s }, za0h.s[x12]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc0860420  // mova { z0.s-z3.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860468  // mova { z8.s-z11.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840582  // mova za2h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa043c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c5b8  // st1w { z24.s-z27.s }, pn9.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5d8  // st1w { z24.s-z27.s }, pn9.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c5c0  // st1w { z0.s-z3.s }, pn9.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c5dc  // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c5a0  // st1w { z0.s-z3.s }, pn9.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 23f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa060c5a0  // st1w { z0.s-z3.s }, pn9.b, [x13]\n"
+      ".inst 0xa060c5c0  // st1w { z0.s-z3.s }, pn9.b, [x14]\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860464  // mova { z4.s-z7.s }, za3h.s[x12]\n"
-      ".inst 0xa061c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5dc  // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5c4  // st1w { z4.s-z7.s }, pn9.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 23f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "sub x24, x11, x10\n"
-      "cntw x23\n"
-      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
-      "cmp x24, x23\n"
-      "csel x21, x24, x23, LT\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "lsr x20, x21, #0x2\n"
-      "madd x25, x10, x22, x25\n"  // C += m * ldc
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "sub x25, x13, x11\n"
+      "cntw x24\n"
+      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
+      "cmp x25, x24\n"
+      "csel x22, x25, x24, LT\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "lsr x21, x22, #0x2\n"
+      "madd x26, x11, x23, x26\n"  // C += m * ldc
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0860438  // mova { z24.s-z27.s }, za1h.s[x12]\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604333  // st1w { z19.s, z27.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604353  // st1w { z19.s, z27.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 15b\n"
       "16:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa1604320  // st1w { z0.s, z8.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604340  // st1w { z0.s, z8.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604321  // st1w { z1.s, z9.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604341  // st1w { z1.s, z9.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 17f\n"
-      ".inst 0xa1604322  // st1w { z2.s, z10.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604342  // st1w { z2.s, z10.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "17:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 21f\n"
-      "cmp x24, x23\n"
-      "csel x19, x24, x23, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x24\n"
+      "csel x20, x25, x24, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
-      ".inst 0xa1604330  // st1w { z16.s, z24.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
-      ".inst 0xa1604331  // st1w { z17.s, z25.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604350  // st1w { z16.s, z24.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
+      ".inst 0xa1604351  // st1w { z17.s, z25.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      ".inst 0xa1604332  // st1w { z18.s, z26.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      ".inst 0xa1604333  // st1w { z19.s, z27.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604352  // st1w { z18.s, z26.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      ".inst 0xa1604353  // st1w { z19.s, z27.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 18b\n"
       "19:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa1604324  // st1w { z4.s, z12.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      ".inst 0xa1604344  // st1w { z4.s, z12.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      ".inst 0xa1604325  // st1w { z5.s, z13.s }, p8, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      ".inst 0xa1604345  // st1w { z5.s, z13.s }, p8, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 20f\n"
-      ".inst 0xa1604326  // st1w { z6.s, z14.s }, p8, [x25]\n"
+      ".inst 0xa1604346  // st1w { z6.s, z14.s }, p8, [x26]\n"
       "20:"  // Store to output array: Accumulator row 1 oddments: End
       "21:"  // Store to output array: End
-      "tbz x15, #0, 23f\n"
+      "tbz x16, #0, 23f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "22:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 22b\n"
       "23:"  // End block
-      "incw x9, ALL, MUL #2\n"
-      "cmp x9, x28\n"
-      "blt 3b\n"
       "incw x10, ALL, MUL #2\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "cmp x10, x9\n"
+      "blt 3b\n"
+      "incw x11, ALL, MUL #2\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp
index 3623f5b6c0..d863b6c72a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -84,107 +84,107 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in
   KernelArgs args(A, B, C, ldc, M, N, K, bias, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c1dc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1fc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa041c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa042c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      "whilelt p0.s, x9, x28\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      "whilelt p0.s, x10, x9\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      "ldnt1w { z15.s }, p0/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc09025e0  // addha za0.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e1  // addha za1.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e2  // addha za2.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e3  // addha za3.s, p1/M, p1/M, z15.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19\n"
-      "incw x20, ALL, MUL #4\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "mov x21, x11\n"
+      "incw x20\n"
+      "incw x21, ALL, MUL #4\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa0408350  // ld1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
-      "ldnt1b { z7.b }, p1/Z, [x22]\n"
-      ".inst 0xa041835c  // ld1b { z28.b-z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
-      "ldnt1b { z13.b }, p1/Z, [x22, #1, MUL VL]\n"
-      ".inst 0xa0428340  // ld1b { z0.b-z3.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #2, MUL VL]\n"
-      ".inst 0xa0438358  // ld1b { z24.b-z27.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z23.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa0408370  // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n"
+      "ldnt1b { z7.b }, p1/Z, [x23]\n"
+      ".inst 0xa041837c  // ld1b { z28.b-z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
+      "ldnt1b { z13.b }, p1/Z, [x23, #1, MUL VL]\n"
+      ".inst 0xa0428360  // ld1b { z0.b-z3.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #2, MUL VL]\n"
+      ".inst 0xa0438378  // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z23.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa0872600  // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa0872621  // smopa za1.s, p1/M, p1/M, z17.b, z7.b\n"
       ".inst 0xa0872642  // smopa za2.s, p1/M, p1/M, z18.b, z7.b\n"
       ".inst 0xa0872663  // smopa za3.s, p1/M, p1/M, z19.b, z7.b\n"
-      ".inst 0xa0408350  // ld1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
+      ".inst 0xa0408370  // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n"
       ".inst 0xa08d2780  // smopa za0.s, p1/M, p1/M, z28.b, z13.b\n"
-      "ldnt1b { z7.b }, p1/Z, [x22]\n"
+      "ldnt1b { z7.b }, p1/Z, [x23]\n"
       ".inst 0xa08d27a1  // smopa za1.s, p1/M, p1/M, z29.b, z13.b\n"
       ".inst 0xa08d27c2  // smopa za2.s, p1/M, p1/M, z30.b, z13.b\n"
       ".inst 0xa08d27e3  // smopa za3.s, p1/M, p1/M, z31.b, z13.b\n"
-      ".inst 0xa041835c  // ld1b { z28.b-z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
+      ".inst 0xa041837c  // ld1b { z28.b-z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
       ".inst 0xa08c2400  // smopa za0.s, p1/M, p1/M, z0.b, z12.b\n"
-      "ldnt1b { z13.b }, p1/Z, [x22, #1, MUL VL]\n"
+      "ldnt1b { z13.b }, p1/Z, [x23, #1, MUL VL]\n"
       ".inst 0xa08c2421  // smopa za1.s, p1/M, p1/M, z1.b, z12.b\n"
       ".inst 0xa08c2442  // smopa za2.s, p1/M, p1/M, z2.b, z12.b\n"
       ".inst 0xa08c2463  // smopa za3.s, p1/M, p1/M, z3.b, z12.b\n"
-      ".inst 0xa0428340  // ld1b { z0.b-z3.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #2, MUL VL]\n"
+      ".inst 0xa0428360  // ld1b { z0.b-z3.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #2, MUL VL]\n"
       ".inst 0xa0972700  // smopa za0.s, p1/M, p1/M, z24.b, z23.b\n"
       ".inst 0xa0972721  // smopa za1.s, p1/M, p1/M, z25.b, z23.b\n"
       ".inst 0xa0972742  // smopa za2.s, p1/M, p1/M, z26.b, z23.b\n"
       ".inst 0xa0972763  // smopa za3.s, p1/M, p1/M, z27.b, z23.b\n"
-      ".inst 0xa0438358  // ld1b { z24.b-z27.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z23.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      ".inst 0xa0438378  // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z23.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa0872600  // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n"
@@ -204,237 +204,237 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in
       ".inst 0xa0972742  // smopa za2.s, p1/M, p1/M, z26.b, z23.b\n"
       ".inst 0xa0972763  // smopa za3.s, p1/M, p1/M, z27.b, z23.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa0408350  // ld1b { z16.b-z19.b }, pn8.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #4\n"
-      "ld1b { z7.b }, p1/Z, [x22]\n"
-      "addvl x22, x22, #1\n"
+      ".inst 0xa0408370  // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #4\n"
+      "ld1b { z7.b }, p1/Z, [x23]\n"
+      "addvl x23, x23, #1\n"
       ".inst 0xa0872600  // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n"
       ".inst 0xa0872621  // smopa za1.s, p1/M, p1/M, z17.b, z7.b\n"
       ".inst 0xa0872642  // smopa za2.s, p1/M, p1/M, z18.b, z7.b\n"
       ".inst 0xa0872663  // smopa za3.s, p1/M, p1/M, z19.b, z7.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0860400  // mova { z0.s-z3.s }, za0h.s[x12]\n"
       ".inst 0xc0840680  // mova za0h.s[x12], { z20.s-z23.s }\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa041c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa042c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840683  // mova za3h.s[x12], { z20.s-z23.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c1dc  // st1w { z28.s-z31.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1d0  // st1w { z16.s-z19.s }, pn8.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c1bc  // st1w { z28.s-z31.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1b0  // st1w { z16.s-z19.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 29f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      ".inst 0xa060c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13]\n"
+      ".inst 0xa060c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14]\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
-      ".inst 0xa061c1a4  // st1w { z4.s-z7.s }, pn8.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c1c4  // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1a0  // st1w { z0.s-z3.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1c0  // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 29f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "sub x24, x11, x10\n"
-      "cntw x23\n"
-      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
-      "cmp x24, x23\n"
-      "csel x21, x24, x23, LT\n"
-      "add x25, x25, x9, LSL #2\n"  // C += n
-      "lsr x20, x21, #0x2\n"
-      "madd x25, x10, x22, x25\n"  // C += m * ldc
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "sub x25, x13, x11\n"
+      "cntw x24\n"
+      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
+      "cmp x25, x24\n"
+      "csel x22, x25, x24, LT\n"
+      "add x26, x26, x10, LSL #2\n"  // C += n
+      "lsr x21, x22, #0x2\n"
+      "madd x26, x11, x23, x26\n"  // C += m * ldc
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 16f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 16f\n"
       "15:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
-      "st1w { z28.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "st1w { z29.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z28.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "st1w { z29.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      "st1w { z30.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z31.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z30.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z31.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 15b\n"
       "16:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 17f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 17f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
-      "st1w { z8.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z8.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 17f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z9.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z9.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 17f\n"
-      "st1w { z10.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z10.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "17:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x23\n"
-      "csel x21, x24, x23, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x24\n"
+      "csel x22, x25, x24, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 19f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 19f\n"
       "18:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860420  // mova { z0.s-z3.s }, za1h.s[x12]\n"
-      "st1w { z0.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "st1w { z1.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z0.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "st1w { z1.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      "st1w { z2.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z3.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z2.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z3.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 18b\n"
       "19:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 20f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 20f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 20f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 20f\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "20:"  // Store to output array: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x23\n"
-      "csel x21, x24, x23, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cmp x25, x24\n"
+      "csel x22, x25, x24, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 22f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 22f\n"
       "21:"  // Store to output array: Accumulator row 2 loop
       ".inst 0xc0860450  // mova { z16.s-z19.s }, za2h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      "st1w { z18.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z19.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z18.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z19.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 21b\n"
       "22:"  // Store to output array: Accumulator row 2 oddments
-      "cbz x19, 23f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 23f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860440  // mova { z0.s-z3.s }, za2h.s[x12]\n"
-      "st1w { z0.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z0.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 23f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z1.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z1.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 23f\n"
-      "st1w { z2.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z2.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "23:"  // Store to output array: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 27f\n"
-      "cmp x24, x23\n"
-      "csel x19, x24, x23, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "cmp x25, x24\n"
+      "csel x20, x25, x24, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 25f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 25f\n"
       "24:"  // Store to output array: Accumulator row 3 loop
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      "st1w { z12.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "st1w { z13.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z12.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "st1w { z13.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "add x12, x12, #0x4\n"
-      "st1w { z14.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
-      "cmp x12, x20, LSL #2\n"
-      "st1w { z15.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z14.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
+      "cmp x12, x21, LSL #2\n"
+      "st1w { z15.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "blt 24b\n"
       "25:"  // Store to output array: Accumulator row 3 oddments
-      "cbz x19, 26f\n"
-      "subs x19, x19, #0x1\n"
+      "cbz x20, 26f\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      "st1w { z16.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "st1w { z16.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 26f\n"
-      "subs x19, x19, #0x1\n"
-      "st1w { z17.s }, p0, [x25]\n"
-      "add x25, x25, x22\n"
+      "subs x20, x20, #0x1\n"
+      "st1w { z17.s }, p0, [x26]\n"
+      "add x26, x26, x23\n"
       "beq 26f\n"
-      "st1w { z18.s }, p0, [x25]\n"
+      "st1w { z18.s }, p0, [x26]\n"
       "26:"  // Store to output array: Accumulator row 3 oddments: End
       "27:"  // Store to output array: End
-      "tbz x15, #0, 29f\n"
+      "tbz x16, #0, 29f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "28:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c1cc  // ld1w { z12.s-z15.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1ec  // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840702  // mova za2h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa043c1c8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e8  // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840503  // mova za3h.s[x12], { z8.s-z11.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 28b\n"
       "29:"  // End block
-      "incw x9\n"
-      "cmp x9, x28\n"
+      "incw x10\n"
+      "cmp x10, x9\n"
       "blt 3b\n"
-      "incw x10, ALL, MUL #4\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "incw x11, ALL, MUL #4\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp
index 100f15c7e0..d868ed2b67 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x13, [%x[args], %[offsetof_flags]]\n"
+      "ldr x14, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x11, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x10, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x13, #0, 2f\n"
+      "tbz x14, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc0840580  // mova za0h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa041c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840781  // mova za1h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa042c57c  // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5bc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840782  // mova za2h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa043c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      "addvl x13, x13, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w9, [%x[args], %[offsetof_M]]\n"
+      "ldr w10, [%x[args], %[offsetof_M]]\n"
+      "mov x9, #0x0\n"
       "mov x28, #0x0\n"
-      "mov x27, #0x0\n"
-      "ldr w26, [%x[args], %[offsetof_N]]\n"
-      "ldr x25, [%x[args], %[offsetof_A]]\n"
+      "ldr w27, [%x[args], %[offsetof_N]]\n"
+      "ldr x26, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x24, x25\n"
-      ".inst 0x25ba6770  // whilelt pn8.s, x27, x26, VLx4\n"
-      "tbnz x13, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x25, x26\n"
+      ".inst 0x25bb6790  // whilelt pn8.s, x28, x27, VLx4\n"
+      "tbnz x14, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa01bc279  // ldnt1w { z24.s-z27.s }, p8/Z, [x19, x27, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa01cc299  // ldnt1w { z24.s-z27.s }, p8/Z, [x20, x28, LSL #2]\n"
       ".inst 0xc0902700  // addha za0.s, p1/M, p1/M, z24.s\n"
       ".inst 0xc0902721  // addha za1.s, p1/M, p1/M, z25.s\n"
       ".inst 0xc0902742  // addha za2.s, p1/M, p1/M, z26.s\n"
       ".inst 0xc0902763  // addha za3.s, p1/M, p1/M, z27.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x27\n"
       "mov x20, x28\n"
-      "incw x19, ALL, MUL #4\n"
-      "incw x20\n"
-      "cmp x19, x26\n"
-      "csel x20, x28, x20, LT\n"
-      "mov x19, x13\n"
-      "bfm x13, XZR, #0x0, #0x0  // bfc x13, #0x0, #0x1\n"
-      "cmp x20, x9\n"
-      "csel x13, x19, x13, LT\n"
+      "mov x21, x9\n"
+      "incw x20, ALL, MUL #4\n"
+      "incw x21\n"
+      "cmp x20, x27\n"
+      "csel x21, x9, x21, LT\n"
+      "mov x20, x14\n"
+      "bfm x14, XZR, #0x0, #0x0  // bfc x14, #0x0, #0x1\n"
+      "cmp x21, x10\n"
+      "csel x14, x20, x14, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x27, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      "ld1b { z10.b }, p1/Z, [x24]\n"
-      ".inst 0xa04086dd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
-      "ld1b { z16.b }, p1/Z, [x24, #1, MUL VL]\n"
-      ".inst 0xa04186cd  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      "ld1b { z21.b }, p1/Z, [x24, #2, MUL VL]\n"
-      ".inst 0xa04286d9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
-      "ld1b { z19.b }, p1/Z, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
-      ".inst 0xa04386c1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x28, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
+      ".inst 0xa04086fd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
+      "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n"
+      ".inst 0xa04186ed  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n"
+      ".inst 0xa04286f9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
+      "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
+      ".inst 0xa04386e1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa1bc2540  // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa1bd2541  // umopa za1.s, p1/M, p1/M, z10.b, z29.b\n"
       ".inst 0xa1be2542  // umopa za2.s, p1/M, p1/M, z10.b, z30.b\n"
       ".inst 0xa1bf2543  // umopa za3.s, p1/M, p1/M, z10.b, z31.b\n"
-      "ld1b { z10.b }, p1/Z, [x24]\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
       ".inst 0xa1ac2600  // umopa za0.s, p1/M, p1/M, z16.b, z12.b\n"
-      ".inst 0xa04086dd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa04086fd  // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa1ad2601  // umopa za1.s, p1/M, p1/M, z16.b, z13.b\n"
       ".inst 0xa1ae2602  // umopa za2.s, p1/M, p1/M, z16.b, z14.b\n"
       ".inst 0xa1af2603  // umopa za3.s, p1/M, p1/M, z16.b, z15.b\n"
-      "ld1b { z16.b }, p1/Z, [x24, #1, MUL VL]\n"
+      "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n"
       ".inst 0xa1b826a0  // umopa za0.s, p1/M, p1/M, z21.b, z24.b\n"
-      ".inst 0xa04186cd  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa04186ed  // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa1b926a1  // umopa za1.s, p1/M, p1/M, z21.b, z25.b\n"
       ".inst 0xa1ba26a2  // umopa za2.s, p1/M, p1/M, z21.b, z26.b\n"
       ".inst 0xa1bb26a3  // umopa za3.s, p1/M, p1/M, z21.b, z27.b\n"
-      "ld1b { z21.b }, p1/Z, [x24, #2, MUL VL]\n"
-      ".inst 0xa04286d9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x22, #0x8, MUL VL]\n"
+      "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n"
+      ".inst 0xa04286f9  // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n"
       ".inst 0xa1a02660  // umopa za0.s, p1/M, p1/M, z19.b, z0.b\n"
       ".inst 0xa1a12661  // umopa za1.s, p1/M, p1/M, z19.b, z1.b\n"
       ".inst 0xa1a22662  // umopa za2.s, p1/M, p1/M, z19.b, z2.b\n"
       ".inst 0xa1a32663  // umopa za3.s, p1/M, p1/M, z19.b, z3.b\n"
-      "ld1b { z19.b }, p1/Z, [x24, #3, MUL VL]\n"
-      "addvl x24, x24, #4\n"
-      ".inst 0xa04386c1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x22, #0xc, MUL VL]\n"
-      "addvl x22, x22, #16\n"
+      "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n"
+      "addvl x25, x25, #4\n"
+      ".inst 0xa04386e1  // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n"
+      "addvl x23, x23, #16\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa1bc2540  // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
@@ -210,76 +210,76 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
       ".inst 0xa1a22662  // umopa za2.s, p1/M, p1/M, z19.b, z2.b\n"
       ".inst 0xa1a32663  // umopa za3.s, p1/M, p1/M, z19.b, z3.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      "ld1b { z10.b }, p1/Z, [x24]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x24, x24, #1\n"
-      ".inst 0xa04086dc  // ld1b { z28.b-z31.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #4\n"
+      "ld1b { z10.b }, p1/Z, [x25]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x25, x25, #1\n"
+      ".inst 0xa04086fc  // ld1b { z28.b-z31.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #4\n"
       ".inst 0xa1bc2540  // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n"
       ".inst 0xa1bd2541  // umopa za1.s, p1/M, p1/M, z10.b, z29.b\n"
       ".inst 0xa1be2542  // umopa za2.s, p1/M, p1/M, z10.b, z30.b\n"
       ".inst 0xa1bf2543  // umopa za3.s, p1/M, p1/M, z10.b, z31.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      "ld1w { z14.s }, p1/Z, [x24]\n"
-      "addvl x24, x24, #1\n"
+      "ld1w { z14.s }, p1/Z, [x25]\n"
+      "addvl x25, x25, #1\n"
       ".inst 0xc09125c0  // addva za0.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c1  // addva za1.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c2  // addva za2.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c3  // addva za3.s, p1/M, p1/M, z14.s\n"
-      "tbz x13, #1, 14f\n"
-      "tbz x13, #0, 12f\n"
+      "tbz x14, #1, 14f\n"
+      "tbz x14, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c578  // ld1w { z24.s-z27.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5b8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc0840700  // mova za0h.s[x12], { z24.s-z27.s }\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa041c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
       ".inst 0xc0860458  // mova { z24.s-z27.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa042c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c564  // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5a4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c55c  // st1w { z28.s-z31.s }, pn9.b, [x10]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c57c  // st1w { z28.s-z31.s }, pn9.b, [x11]\n"
+      "addvl x13, x13, #16\n"
+      ".inst 0xa061c568  // st1w { z8.s-z11.s }, pn9.b, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa062c578  // st1w { z24.s-z27.s }, pn9.b, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa063c56c  // st1w { z12.s-z15.s }, pn9.b, [x11, #0xc, MUL VL]\n"
       "addvl x11, x11, #16\n"
-      ".inst 0xa061c548  // st1w { z8.s-z11.s }, pn9.b, [x10, #0x4, MUL VL]\n"
-      ".inst 0xa062c558  // st1w { z24.s-z27.s }, pn9.b, [x10, #0x8, MUL VL]\n"
-      ".inst 0xa063c54c  // st1w { z12.s-z15.s }, pn9.b, [x10, #0xc, MUL VL]\n"
-      "addvl x10, x10, #16\n"
       "blt 11b\n"
       "b 21f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc0860420  // mova { z0.s-z3.s }, za1h.s[x12]\n"
-      ".inst 0xa060c55c  // st1w { z28.s-z31.s }, pn9.b, [x10]\n"
+      ".inst 0xa060c57c  // st1w { z28.s-z31.s }, pn9.b, [x11]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa061c540  // st1w { z0.s-z3.s }, pn9.b, [x10, #0x4, MUL VL]\n"
+      ".inst 0xa061c560  // st1w { z0.s-z3.s }, pn9.b, [x11, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c548  // st1w { z8.s-z11.s }, pn9.b, [x10, #0x8, MUL VL]\n"
-      ".inst 0xa063c550  // st1w { z16.s-z19.s }, pn9.b, [x10, #0xc, MUL VL]\n"
-      "addvl x10, x10, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c568  // st1w { z8.s-z11.s }, pn9.b, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa063c570  // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n"
+      "addvl x11, x11, #16\n"
       "blt 13b\n"
       "b 21f\n"
       "14:"  // Store to output array
-      "ldr x23, [%x[args], %[offsetof_C]]\n"
-      "add x23, x23, x27\n"  // C += n
-      "sub x22, x9, x28\n"
+      "ldr x24, [%x[args], %[offsetof_C]]\n"
+      "add x24, x24, x28\n"  // C += n
+      "sub x23, x10, x9\n"
       "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x21, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x23, x28, x21, x23\n"  // C += m * ldc
+      "ldr x22, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x24, x9, x22, x24\n"  // C += m * ldc
       "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
@@ -290,24 +290,24 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
       "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x13, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x27\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa040c26c  // ld1w { z12.s-z15.s }, p8/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa040c264  // ld1w { z4.s-z7.s }, p8/Z, [x19]\n"
+      "tbz x14, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x28\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa040c28c  // ld1w { z12.s-z15.s }, p8/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa040c284  // ld1w { z4.s-z7.s }, p8/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x19\n"
-      "whilelt p0.b, x27, x26\n"
-      "cmp x22, x19\n"
-      "csel x19, x22, x19, LT\n"
-      "lsr x20, x19, #0x1\n"
+      "cntw x20\n"
+      "whilelt p0.b, x28, x27\n"
+      "cmp x23, x20\n"
+      "csel x20, x23, x20, LT\n"
+      "lsr x21, x20, #0x1\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x1\n"
-      "cbz x20, 17f\n"
+      "and x20, x20, #0x1\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086001a  // mova { z26.s-z27.s }, za0h.s[x12, 0:1]\n"
       ".inst 0xc086005c  // mova { z28.s-z29.s }, za1h.s[x12, 0:1]\n"
@@ -317,7 +317,7 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
       ".inst 0xc1ada41c  // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n"
       ".inst 0xc1aea416  // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n"
       "add x12, x12, #0x2\n"
-      "cmp x12, x20, LSL #1\n"
+      "cmp x12, x21, LSL #1\n"
       ".inst 0xc1afa410  // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n"
       ".inst 0xc1a4a23a  // srshl { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n"
       ".inst 0xc1a5a23c  // srshl { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n"
@@ -336,14 +336,14 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
       "uzp1 z18.b, z27.b, z29.b\n"
       "uzp1 z17.b, z23.b, z17.b\n"
       "uzp1 z16.b, z19.b, z16.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
-      "add x23, x23, x21\n"
+      "st1b { z16.b }, p0, [x24]\n"
+      "add x24, x24, x22\n"
       "uzp1 z16.b, z18.b, z17.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
-      "add x23, x23, x21\n"
+      "st1b { z16.b }, p0, [x24]\n"
+      "add x24, x24, x22\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc0860002  // mova { z2.s-z3.s }, za0h.s[x12, 0:1]\n"
       ".inst 0xc0860058  // mova { z24.s-z25.s }, za1h.s[x12, 0:1]\n"
       ".inst 0xc1aca402  // sqdmulh { z2.s-z3.s }, { z2.s-z3.s }, z12.s\n"
@@ -367,38 +367,38 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin
       ".inst 0xc1b4c6aa  // sclamp { z10.s-z11.s }, z21.s, z20.s\n"
       "uzp1 z16.b, z16.b, z10.b\n"
       "uzp1 z16.b, z23.b, z16.b\n"
-      "st1b { z16.b }, p0, [x23]\n"
+      "st1b { z16.b }, p0, [x24]\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
       "19:"  // Store to output array: End
-      "tbz x13, #0, 21f\n"
+      "tbz x14, #0, 21f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "20:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11]\n"
+      ".inst 0xa040c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n"
+      ".inst 0xa041c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c570  // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n"
+      ".inst 0xa042c5b0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c56c  // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n"
+      ".inst 0xa043c5ac  // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x11, x11, #16\n"
+      "cmp x12, x20\n"
+      "addvl x13, x13, #16\n"
       "blt 20b\n"
       "21:"  // End block
-      "incw x27, ALL, MUL #4\n"
-      "cmp x27, x26\n"
+      "incw x28, ALL, MUL #4\n"
+      "cmp x28, x27\n"
       "blt 3b\n"
-      "incw x28\n"
-      "cmp x28, x9\n"
-      "mov x27, #0x0\n"
-      "mov x25, x24\n"
+      "incw x9\n"
+      "cmp x9, x10\n"
+      "mov x28, #0x0\n"
+      "mov x26, x25\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp
index 6c42012482..cb0e9521e3 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207811  // ptrue pn9.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840400  // mova za0h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa041c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c5c0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5e0  // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840402  // mova za2h.s[x12], { z0.s-z3.s }\n"
-      ".inst 0xa043c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840783  // mova za3h.s[x12], { z28.s-z31.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      ".inst 0x25bc4530  // whilelt pn8.s, x9, x28, VLx2\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      ".inst 0x25a94550  // whilelt pn8.s, x10, x9, VLx2\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      ".inst 0xa0094275  // ldnt1w { z20.s-z21.s }, p8/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      ".inst 0xa00a4295  // ldnt1w { z20.s-z21.s }, p8/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc0902680  // addha za0.s, p1/M, p1/M, z20.s\n"
       ".inst 0xc09026a1  // addha za1.s, p1/M, p1/M, z21.s\n"
       ".inst 0xc0902682  // addha za2.s, p1/M, p1/M, z20.s\n"
       ".inst 0xc09026a3  // addha za3.s, p1/M, p1/M, z21.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19, ALL, MUL #2\n"
+      "mov x21, x11\n"
       "incw x20, ALL, MUL #2\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "incw x21, ALL, MUL #2\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
-      ".inst 0xa04006d1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
-      ".inst 0xa041074e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
-      ".inst 0xa04106c9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
-      ".inst 0xa0420740  // ld1b { z0.b-z1.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa14206dc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306ca  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
+      ".inst 0xa04006f1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
+      ".inst 0xa041076e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
+      ".inst 0xa04106e9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
+      ".inst 0xa0420760  // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa14206fc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306ea  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa1b027c0  // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa1b127c1  // umopa za1.s, p1/M, p1/M, z30.b, z17.b\n"
       ".inst 0xa1b027e2  // umopa za2.s, p1/M, p1/M, z31.b, z16.b\n"
       ".inst 0xa1b127e3  // umopa za3.s, p1/M, p1/M, z31.b, z17.b\n"
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
       ".inst 0xa1a825c0  // umopa za0.s, p1/M, p1/M, z14.b, z8.b\n"
-      ".inst 0xa04006d1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
+      ".inst 0xa04006f1  // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
       ".inst 0xa1a925c1  // umopa za1.s, p1/M, p1/M, z14.b, z9.b\n"
       ".inst 0xa1a825e2  // umopa za2.s, p1/M, p1/M, z15.b, z8.b\n"
       ".inst 0xa1a925e3  // umopa za3.s, p1/M, p1/M, z15.b, z9.b\n"
-      ".inst 0xa041074e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x26, #0x2, MUL VL]\n"
+      ".inst 0xa041076e  // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n"
       ".inst 0xa1b42400  // umopa za0.s, p1/M, p1/M, z0.b, z20.b\n"
-      ".inst 0xa04106c9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x22, #0x2, MUL VL]\n"
+      ".inst 0xa04106e9  // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n"
       ".inst 0xa1bc2401  // umopa za1.s, p1/M, p1/M, z0.b, z28.b\n"
       ".inst 0xa1b42422  // umopa za2.s, p1/M, p1/M, z1.b, z20.b\n"
       ".inst 0xa1bc2423  // umopa za3.s, p1/M, p1/M, z1.b, z28.b\n"
-      ".inst 0xa0420740  // ld1b { z0.b-z1.b }, pn9.b/Z, [x26, #0x4, MUL VL]\n"
-      ".inst 0xa14206dc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x22, #0x4, MUL VL]\n"
+      ".inst 0xa0420760  // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n"
+      ".inst 0xa14206fc  // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n"
       ".inst 0xa1a22480  // umopa za0.s, p1/M, p1/M, z4.b, z2.b\n"
       ".inst 0xa1aa2481  // umopa za1.s, p1/M, p1/M, z4.b, z10.b\n"
       ".inst 0xa1a224a2  // umopa za2.s, p1/M, p1/M, z5.b, z2.b\n"
       ".inst 0xa1aa24a3  // umopa za3.s, p1/M, p1/M, z5.b, z10.b\n"
-      ".inst 0xa0430744  // ld1b { z4.b-z5.b }, pn9.b/Z, [x26, #0x6, MUL VL]\n"
-      "addvl x26, x26, #8\n"
-      ".inst 0xa14306ca  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x22, #0x6, MUL VL]\n"
-      "addvl x22, x22, #8\n"
+      ".inst 0xa0430764  // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n"
+      "addvl x27, x27, #8\n"
+      ".inst 0xa14306ea  // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n"
+      "addvl x23, x23, #8\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa1b027c0  // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
@@ -210,107 +210,107 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
       ".inst 0xa1a224a2  // umopa za2.s, p1/M, p1/M, z5.b, z2.b\n"
       ".inst 0xa1aa24a3  // umopa za3.s, p1/M, p1/M, z5.b, z10.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa040075e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #2\n"
-      ".inst 0xa04006d0  // ld1b { z16.b-z17.b }, pn9.b/Z, [x22]\n"
-      "addvl x22, x22, #2\n"
+      ".inst 0xa040077e  // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #2\n"
+      ".inst 0xa04006f0  // ld1b { z16.b-z17.b }, pn9.b/Z, [x23]\n"
+      "addvl x23, x23, #2\n"
       ".inst 0xa1b027c0  // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n"
       ".inst 0xa1b127c1  // umopa za1.s, p1/M, p1/M, z30.b, z17.b\n"
       ".inst 0xa1b027e2  // umopa za2.s, p1/M, p1/M, z31.b, z16.b\n"
       ".inst 0xa1b127e3  // umopa za3.s, p1/M, p1/M, z31.b, z17.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      ".inst 0xa040474e  // ld1w { z14.s-z15.s }, pn9.b/Z, [x26]\n"
-      "addvl x26, x26, #2\n"
+      ".inst 0xa040476e  // ld1w { z14.s-z15.s }, pn9.b/Z, [x27]\n"
+      "addvl x27, x27, #2\n"
       ".inst 0xc09125c0  // addva za0.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125c1  // addva za1.s, p1/M, p1/M, z14.s\n"
       ".inst 0xc09125e2  // addva za2.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09125e3  // addva za3.s, p1/M, p1/M, z15.s\n"
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c5dc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5fc  // ld1w { z28.s-z31.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0860408  // mova { z8.s-z11.s }, za0h.s[x12]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
       ".inst 0xc0860434  // mova { z20.s-z23.s }, za1h.s[x12]\n"
-      ".inst 0xa041c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840701  // mova za1h.s[x12], { z24.s-z27.s }\n"
       ".inst 0xc086045c  // mova { z28.s-z31.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
-      ".inst 0xa042c5d8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f8  // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840702  // mova za2h.s[x12], { z24.s-z27.s }\n"
-      ".inst 0xa043c5cc  // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5ec  // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840583  // mova za3h.s[x12], { z12.s-z15.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c5d4  // st1w { z20.s-z23.s }, pn9.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c5dc  // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c5b4  // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c5bc  // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 24f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0860424  // mova { z4.s-z7.s }, za1h.s[x12]\n"
-      ".inst 0xa060c5b0  // st1w { z16.s-z19.s }, pn9.b, [x13]\n"
+      ".inst 0xa060c5d0  // st1w { z16.s-z19.s }, pn9.b, [x14]\n"
       ".inst 0xc0860448  // mova { z8.s-z11.s }, za2h.s[x12]\n"
       ".inst 0xc086046c  // mova { z12.s-z15.s }, za3h.s[x12]\n"
-      ".inst 0xa061c5a4  // st1w { z4.s-z7.s }, pn9.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c5c4  // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c5a8  // st1w { z8.s-z11.s }, pn9.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c5ac  // st1w { z12.s-z15.s }, pn9.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c5c8  // st1w { z8.s-z11.s }, pn9.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c5cc  // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 24f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9\n"  // C += n
-      "sub x24, x11, x10\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10\n"  // C += n
+      "sub x25, x13, x11\n"
       "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
       "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
       "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z11.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z25.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z24.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x15, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x9\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa0404262  // ld1w { z2.s-z3.s }, p8/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      ".inst 0xa0404260  // ld1w { z0.s-z1.s }, p8/Z, [x19]\n"
+      "tbz x16, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x10\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa0404282  // ld1w { z2.s-z3.s }, p8/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      ".inst 0xa0404280  // ld1w { z0.s-z1.s }, p8/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x22\n"
-      "whilelt p0.h, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cntw x23\n"
+      "whilelt p0.h, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 17f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
       ".inst 0xc1a2ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n"
       ".inst 0xc1a3ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z3.s\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a0aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n"
       ".inst 0xc1a1aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n"
       ".inst 0xc1abab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n"
@@ -318,25 +318,25 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
       ".inst 0xc1b8cf2c  // sclamp { z12.s-z15.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf3c  // sclamp { z28.s-z31.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z12.h, z28.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z13.h, z29.h\n"
       "uzp1 z17.h, z14.h, z30.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z15.h, z31.h\n"
-      "st1b { z17.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z17.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc086041c  // mova { z28.s-z31.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
       ".inst 0xc1a2ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n"
       ".inst 0xc1a3ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a0aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n"
       ".inst 0xc1a1aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n"
       ".inst 0xc1abab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n"
@@ -344,34 +344,34 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
       ".inst 0xc1b8cf3c  // sclamp { z28.s-z31.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf2c  // sclamp { z12.s-z15.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z28.h, z12.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       "uzp1 z16.h, z29.h, z13.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
       "uzp1 z16.h, z30.h, z14.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 22f\n"
-      "whilelt p0.h, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "whilelt p0.h, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860444  // mova { z4.s-z7.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
       ".inst 0xc1a2ac04  // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n"
       ".inst 0xc1a3ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a0aa24  // srshl { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n"
       ".inst 0xc1a1aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n"
       ".inst 0xc1abab04  // add { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n"
@@ -379,25 +379,25 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
       ".inst 0xc1b8cf24  // sclamp { z4.s-z7.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf30  // sclamp { z16.s-z19.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z4.h, z16.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z5.h, z17.h\n"
       "uzp1 z17.h, z6.h, z18.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "uzp1 z16.h, z7.h, z19.h\n"
-      "st1b { z17.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z17.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860470  // mova { z16.s-z19.s }, za3h.s[x12]\n"
       ".inst 0xc1a2ac14  // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z2.s\n"
       ".inst 0xc1a3ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a0aa34  // srshl { z20.s-z23.s }, { z20.s-z23.s }, z0.s\n"
       ".inst 0xc1a1aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n"
       ".inst 0xc1abab14  // add { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n"
@@ -405,47 +405,47 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin
       ".inst 0xc1b8cf34  // sclamp { z20.s-z23.s }, z25.s, z24.s\n"
       ".inst 0xc1b8cf30  // sclamp { z16.s-z19.s }, z25.s, z24.s\n"
       "uzp1 z16.h, z20.h, z16.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       "uzp1 z16.h, z21.h, z17.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.h }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
       "uzp1 z16.h, z22.h, z18.h\n"
-      "st1b { z16.h }, p0, [x25]\n"
+      "st1b { z16.h }, p0, [x26]\n"
       "21:"  // Store to output array: Accumulator row 1 oddments: End
       "22:"  // Store to output array: End
-      "tbz x15, #0, 24f\n"
+      "tbz x16, #0, 24f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "23:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n"
+      ".inst 0xa040c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n"
       ".inst 0xc0840600  // mova za0h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa041c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c5d0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c5f0  // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c5c4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c5e4  // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 23b\n"
       "24:"  // End block
-      "incw x9, ALL, MUL #2\n"
-      "cmp x9, x28\n"
-      "blt 3b\n"
       "incw x10, ALL, MUL #2\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "cmp x10, x9\n"
+      "blt 3b\n"
+      "incw x11, ALL, MUL #2\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp
index 40d2fff8c2..8f8886b876 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef __ARM_FEATURE_SVE
 #ifdef ARM_COMPUTE_ENABLE_SME2
@@ -90,107 +90,107 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin
   KernelArgs args(A, B, C, ldc, M, N, K, bias, rq, n_0, accumulate, accumulator_buffer);
 
   __asm__ __volatile__(
-      "ldr x15, [%x[args], %[offsetof_flags]]\n"
+      "ldr x16, [%x[args], %[offsetof_flags]]\n"
       ".inst 0xd503477f  // SMSTART ZA\n"
       "ptrue p1.b\n"
       ".inst 0x25207810  // ptrue pn8.b\n"
+      "ldr x15, [%x[args], %[offsetof_accumulator_buffer]]\n"
       "ldr x14, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "ldr x13, [%x[args], %[offsetof_accumulator_buffer]]\n"
-      "tbz x15, #0, 2f\n"
+      "tbz x16, #0, 2f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "1:"  // Initial accumulator load from buffer: Loop
-      ".inst 0xa040c1dc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1fc  // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840780  // mova za0h.s[x12], { z28.s-z31.s }\n"
-      ".inst 0xa041c1cc  // ld1w { z12.s-z15.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1ec  // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840581  // mova za1h.s[x12], { z12.s-z15.s }\n"
-      ".inst 0xa042c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840682  // mova za2h.s[x12], { z20.s-z23.s }\n"
-      ".inst 0xa043c1d8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1f8  // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840703  // mova za3h.s[x12], { z24.s-z27.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 1b\n"
       "2:"  // Initial accumulator load from buffer: End
-      "ldr w11, [%x[args], %[offsetof_M]]\n"
+      "ldr w13, [%x[args], %[offsetof_M]]\n"
+      "mov x11, #0x0\n"
       "mov x10, #0x0\n"
-      "mov x9, #0x0\n"
-      "ldr w28, [%x[args], %[offsetof_N]]\n"
-      "ldr x27, [%x[args], %[offsetof_A]]\n"
+      "ldr w9, [%x[args], %[offsetof_N]]\n"
+      "ldr x28, [%x[args], %[offsetof_A]]\n"
       "3:"  // M and N loop
-      "mov x26, x27\n"
-      "whilelt p0.s, x9, x28\n"
-      "tbnz x15, #0, 4f\n"
-      "ldr x19, [%x[args], %[offsetof_bias]]\n"
+      "mov x27, x28\n"
+      "whilelt p0.s, x10, x9\n"
+      "tbnz x16, #0, 4f\n"
+      "ldr x20, [%x[args], %[offsetof_bias]]\n"
       ".inst 0xc00800ff  // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n"
-      "cbz x19, 5f\n"
-      "ldnt1w { z15.s }, p0/Z, [x19, x9, LSL #2]\n"
+      "cbz x20, 5f\n"
+      "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n"
       ".inst 0xc09025e0  // addha za0.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e1  // addha za1.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e2  // addha za2.s, p1/M, p1/M, z15.s\n"
       ".inst 0xc09025e3  // addha za3.s, p1/M, p1/M, z15.s\n"
       "4:"  // Prepare accumulators: Test for last block
-      "mov x19, x9\n"
       "mov x20, x10\n"
-      "incw x19\n"
-      "incw x20, ALL, MUL #4\n"
-      "cmp x19, x28\n"
-      "csel x20, x10, x20, LT\n"
-      "mov x19, x15\n"
-      "bfm x15, XZR, #0x0, #0x0  // bfc x15, #0x0, #0x1\n"
-      "cmp x20, x11\n"
-      "csel x15, x19, x15, LT\n"
+      "mov x21, x11\n"
+      "incw x20\n"
+      "incw x21, ALL, MUL #4\n"
+      "cmp x20, x9\n"
+      "csel x21, x11, x21, LT\n"
+      "mov x20, x16\n"
+      "bfm x16, XZR, #0x0, #0x0  // bfc x16, #0x0, #0x1\n"
+      "cmp x21, x13\n"
+      "csel x16, x20, x16, LT\n"
       "5:"  // Prepare accumulators: End
-      "ldr x19, [%x[args], %[offsetof_K]]\n"
-      "add x19, x19, #0x3\n"
-      "lsr x19, x19, #0x2\n"
-      "ldr x22, [%x[args], %[offsetof_B]]\n"
-      "lsr x21, x19, #0x2\n"
-      "and x20, x19, #0x3\n"
-      "ldr x19, [%x[args], %[offsetof_kstride_bytes]]\n"
-      "madd x22, x9, x19, x22\n"  // bptr = B + n * kstride_bytes
-      "cbz x21, 8f\n"
-      "subs x21, x21, #0x1\n"
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
-      "ldnt1b { z0.b }, p1/Z, [x22]\n"
-      ".inst 0xa1418353  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
-      "ldnt1b { z9.b }, p1/Z, [x22, #1, MUL VL]\n"
-      ".inst 0xa1428350  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z21.b }, p1/Z, [x22, #2, MUL VL]\n"
-      ".inst 0xa1438342  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      "ldr x20, [%x[args], %[offsetof_K]]\n"
+      "add x20, x20, #0x3\n"
+      "lsr x20, x20, #0x2\n"
+      "ldr x23, [%x[args], %[offsetof_B]]\n"
+      "lsr x22, x20, #0x2\n"
+      "and x21, x20, #0x3\n"
+      "ldr x20, [%x[args], %[offsetof_kstride_bytes]]\n"
+      "madd x23, x10, x20, x23\n"  // bptr = B + n * kstride_bytes
+      "cbz x22, 8f\n"
+      "subs x22, x22, #0x1\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
+      "ldnt1b { z0.b }, p1/Z, [x23]\n"
+      ".inst 0xa1418373  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
+      "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n"
+      ".inst 0xa1428370  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n"
+      ".inst 0xa1438362  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "ble 7f\n"
       "6:"  // K loop
       ".inst 0xa1a02640  // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
-      "subs x21, x21, #0x1\n"
+      "subs x22, x22, #0x1\n"
       ".inst 0xa1a026c1  // umopa za1.s, p1/M, p1/M, z22.b, z0.b\n"
       ".inst 0xa1a02742  // umopa za2.s, p1/M, p1/M, z26.b, z0.b\n"
       ".inst 0xa1a027c3  // umopa za3.s, p1/M, p1/M, z30.b, z0.b\n"
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
       ".inst 0xa1a92660  // umopa za0.s, p1/M, p1/M, z19.b, z9.b\n"
-      "ldnt1b { z0.b }, p1/Z, [x22]\n"
+      "ldnt1b { z0.b }, p1/Z, [x23]\n"
       ".inst 0xa1a926e1  // umopa za1.s, p1/M, p1/M, z23.b, z9.b\n"
       ".inst 0xa1a92762  // umopa za2.s, p1/M, p1/M, z27.b, z9.b\n"
       ".inst 0xa1a927e3  // umopa za3.s, p1/M, p1/M, z31.b, z9.b\n"
-      ".inst 0xa1418353  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n"
+      ".inst 0xa1418373  // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n"
       ".inst 0xa1b52600  // umopa za0.s, p1/M, p1/M, z16.b, z21.b\n"
-      "ldnt1b { z9.b }, p1/Z, [x22, #1, MUL VL]\n"
+      "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n"
       ".inst 0xa1b52681  // umopa za1.s, p1/M, p1/M, z20.b, z21.b\n"
       ".inst 0xa1b52702  // umopa za2.s, p1/M, p1/M, z24.b, z21.b\n"
       ".inst 0xa1b52783  // umopa za3.s, p1/M, p1/M, z28.b, z21.b\n"
-      ".inst 0xa1428350  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n"
-      "ldnt1b { z21.b }, p1/Z, [x22, #2, MUL VL]\n"
+      ".inst 0xa1428370  // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n"
+      "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n"
       ".inst 0xa1ac2440  // umopa za0.s, p1/M, p1/M, z2.b, z12.b\n"
       ".inst 0xa1ac24c1  // umopa za1.s, p1/M, p1/M, z6.b, z12.b\n"
       ".inst 0xa1ac2542  // umopa za2.s, p1/M, p1/M, z10.b, z12.b\n"
       ".inst 0xa1ac25c3  // umopa za3.s, p1/M, p1/M, z14.b, z12.b\n"
-      ".inst 0xa1438342  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n"
-      "addvl x26, x26, #16\n"
-      "ldnt1b { z12.b }, p1/Z, [x22, #3, MUL VL]\n"
-      "addvl x22, x22, #4\n"
+      ".inst 0xa1438362  // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n"
+      "addvl x27, x27, #16\n"
+      "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n"
+      "addvl x23, x23, #4\n"
       "bgt 6b\n"
       "7:"  // K loop tail
       ".inst 0xa1a02640  // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
@@ -210,294 +210,294 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin
       ".inst 0xa1ac2542  // umopa za2.s, p1/M, p1/M, z10.b, z12.b\n"
       ".inst 0xa1ac25c3  // umopa za3.s, p1/M, p1/M, z14.b, z12.b\n"
       "8:"  // K oddments
-      "cbz x20, 10f\n"
+      "cbz x21, 10f\n"
       "9:"  // K oddments: Loop
-      ".inst 0xa1408352  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x26]\n"
-      "subs x20, x20, #0x1\n"
-      "addvl x26, x26, #4\n"
-      "ld1b { z0.b }, p1/Z, [x22]\n"
-      "addvl x22, x22, #1\n"
+      ".inst 0xa1408372  // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n"
+      "subs x21, x21, #0x1\n"
+      "addvl x27, x27, #4\n"
+      "ld1b { z0.b }, p1/Z, [x23]\n"
+      "addvl x23, x23, #1\n"
       ".inst 0xa1a02640  // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n"
       ".inst 0xa1a026c1  // umopa za1.s, p1/M, p1/M, z22.b, z0.b\n"
       ".inst 0xa1a02742  // umopa za2.s, p1/M, p1/M, z26.b, z0.b\n"
       ".inst 0xa1a027c3  // umopa za3.s, p1/M, p1/M, z30.b, z0.b\n"
       "bgt 9b\n"
       "10:"  // K oddments: End
-      ".inst 0xa040c340  // ld1w { z0.s-z3.s }, pn8.b/Z, [x26]\n"
-      "addvl x26, x26, #4\n"
+      ".inst 0xa040c360  // ld1w { z0.s-z3.s }, pn8.b/Z, [x27]\n"
+      "addvl x27, x27, #4\n"
       ".inst 0xc0912400  // addva za0.s, p1/M, p1/M, z0.s\n"
       ".inst 0xc0912421  // addva za1.s, p1/M, p1/M, z1.s\n"
       ".inst 0xc0912442  // addva za2.s, p1/M, p1/M, z2.s\n"
       ".inst 0xc0912463  // addva za3.s, p1/M, p1/M, z3.s\n"
-      "tbz x15, #1, 14f\n"
-      "tbz x15, #0, 12f\n"
+      "tbz x16, #1, 14f\n"
+      "tbz x16, #0, 12f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "11:"  // Store to partial result buffer: Store and refill: Loop
-      ".inst 0xa040c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc0840680  // mova za0h.s[x12], { z20.s-z23.s }\n"
       ".inst 0xc0860428  // mova { z8.s-z11.s }, za1h.s[x12]\n"
-      ".inst 0xa041c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840481  // mova za1h.s[x12], { z4.s-z7.s }\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc086047c  // mova { z28.s-z31.s }, za3h.s[x12]\n"
-      ".inst 0xa042c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840482  // mova za2h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa043c1d4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1f4  // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840683  // mova za3h.s[x12], { z20.s-z23.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa060c1b0  // st1w { z16.s-z19.s }, pn8.b, [x13]\n"
+      "cmp x12, x20\n"
+      ".inst 0xa060c1d0  // st1w { z16.s-z19.s }, pn8.b, [x14]\n"
+      "addvl x15, x15, #16\n"
+      ".inst 0xa061c1c8  // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa062c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1dc  // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n"
       "addvl x14, x14, #16\n"
-      ".inst 0xa061c1a8  // st1w { z8.s-z11.s }, pn8.b, [x13, #0x4, MUL VL]\n"
-      ".inst 0xa062c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1bc  // st1w { z28.s-z31.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
       "blt 11b\n"
       "b 30f\n"
       "12:"  // Store to partial result buffer: Store only
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "13:"  // Store to partial result buffer: Store only: Loop
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc086042c  // mova { z12.s-z15.s }, za1h.s[x12]\n"
-      ".inst 0xa060c1b0  // st1w { z16.s-z19.s }, pn8.b, [x13]\n"
+      ".inst 0xa060c1d0  // st1w { z16.s-z19.s }, pn8.b, [x14]\n"
       ".inst 0xc0860454  // mova { z20.s-z23.s }, za2h.s[x12]\n"
       ".inst 0xc0860478  // mova { z24.s-z27.s }, za3h.s[x12]\n"
-      ".inst 0xa061c1ac  // st1w { z12.s-z15.s }, pn8.b, [x13, #0x4, MUL VL]\n"
+      ".inst 0xa061c1cc  // st1w { z12.s-z15.s }, pn8.b, [x14, #0x4, MUL VL]\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      ".inst 0xa062c1b4  // st1w { z20.s-z23.s }, pn8.b, [x13, #0x8, MUL VL]\n"
-      ".inst 0xa063c1b8  // st1w { z24.s-z27.s }, pn8.b, [x13, #0xc, MUL VL]\n"
-      "addvl x13, x13, #16\n"
+      "cmp x12, x20\n"
+      ".inst 0xa062c1d4  // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa063c1d8  // st1w { z24.s-z27.s }, pn8.b, [x14, #0xc, MUL VL]\n"
+      "addvl x14, x14, #16\n"
       "blt 13b\n"
       "b 30f\n"
       "14:"  // Store to output array
-      "ldr x25, [%x[args], %[offsetof_C]]\n"
-      "add x25, x25, x9\n"  // C += n
-      "sub x24, x11, x10\n"
+      "ldr x26, [%x[args], %[offsetof_C]]\n"
+      "add x26, x26, x10\n"  // C += n
+      "sub x25, x13, x11\n"
       "ld1rw { z8.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n"
-      "ldr x23, [%x[args], %[offsetof_ldcb]]\n"
-      "madd x25, x10, x23, x25\n"  // C += m * ldc
+      "ldr x24, [%x[args], %[offsetof_ldcb]]\n"
+      "madd x26, x11, x24, x26\n"  // C += m * ldc
       "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n"
       "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n"
       "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n"
       "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n"
-      "tbz x15, #2, 15f\n"
-      "ldr w20, [%x[args], %[offsetof_n_0]]\n"
-      "add x20, x20, x9\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      "ld1w { z8.s }, p0/Z, [x19]\n"
-      "ldr x19, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
-      "add x19, x19, x20, LSL #2\n"
-      "ld1w { z7.s }, p0/Z, [x19]\n"
+      "tbz x16, #2, 15f\n"
+      "ldr w21, [%x[args], %[offsetof_n_0]]\n"
+      "add x21, x21, x10\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      "ld1w { z8.s }, p0/Z, [x20]\n"
+      "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n"
+      "add x20, x20, x21, LSL #2\n"
+      "ld1w { z7.s }, p0/Z, [x20]\n"
       "15:"  // Store to output array: Load per-channel parameters: End
-      "cntw x22\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "cntw x23\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 17f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 17f\n"
       "16:"  // Store to output array: Accumulator row 0 loop
       ".inst 0xc086040c  // mova { z12.s-z15.s }, za0h.s[x12]\n"
       ".inst 0xc1a8ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
       ".inst 0xc1a4ccac  // sclamp { z12.s-z15.s }, z5.s, z4.s\n"
-      "st1b { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z15.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z15.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 16b\n"
       "17:"  // Store to output array: Accumulator row 0 oddments
-      "cbz x19, 18f\n"
+      "cbz x20, 18f\n"
       ".inst 0xc0860410  // mova { z16.s-z19.s }, za0h.s[x12]\n"
       ".inst 0xc1a8ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n"
       ".inst 0xc1a6ab10  // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n"
       ".inst 0xc1a4ccb0  // sclamp { z16.s-z19.s }, z5.s, z4.s\n"
-      "st1b { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 18f\n"
-      "st1b { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "18:"  // Store to output array: Accumulator row 0 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 20f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 20f\n"
       "19:"  // Store to output array: Accumulator row 1 loop
       ".inst 0xc0860430  // mova { z16.s-z19.s }, za1h.s[x12]\n"
       ".inst 0xc1a8ac10  // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa30  // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab10  // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n"
       ".inst 0xc1a4ccb0  // sclamp { z16.s-z19.s }, z5.s, z4.s\n"
-      "st1b { z16.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z17.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z18.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z19.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z16.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z17.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z18.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z19.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 19b\n"
       "20:"  // Store to output array: Accumulator row 1 oddments
-      "cbz x19, 21f\n"
+      "cbz x20, 21f\n"
       ".inst 0xc086043c  // mova { z28.s-z31.s }, za1h.s[x12]\n"
       ".inst 0xc1a8ac1c  // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa3c  // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n"
       ".inst 0xc1a6ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n"
       ".inst 0xc1a4ccbc  // sclamp { z28.s-z31.s }, z5.s, z4.s\n"
-      "st1b { z28.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z28.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z29.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z29.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 21f\n"
-      "st1b { z30.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z30.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "21:"  // Store to output array: Accumulator row 1 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x21, x24, x22, LT\n"
-      "lsr x20, x21, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x22, x25, x23, LT\n"
+      "lsr x21, x22, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x21, #0x3\n"
-      "cbz x20, 23f\n"
+      "and x20, x22, #0x3\n"
+      "cbz x21, 23f\n"
       "22:"  // Store to output array: Accumulator row 2 loop
       ".inst 0xc0860458  // mova { z24.s-z27.s }, za2h.s[x12]\n"
       ".inst 0xc1a8ac18  // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa38  // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab18  // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n"
       ".inst 0xc1a4ccb8  // sclamp { z24.s-z27.s }, z5.s, z4.s\n"
-      "st1b { z24.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z25.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z26.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z27.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z24.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z25.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z26.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z27.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 22b\n"
       "23:"  // Store to output array: Accumulator row 2 oddments
-      "cbz x19, 24f\n"
+      "cbz x20, 24f\n"
       ".inst 0xc086044c  // mova { z12.s-z15.s }, za2h.s[x12]\n"
       ".inst 0xc1a8ac0c  // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa2c  // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n"
       ".inst 0xc1a6ab0c  // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n"
       ".inst 0xc1a4ccac  // sclamp { z12.s-z15.s }, z5.s, z4.s\n"
-      "st1b { z12.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z12.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z13.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z13.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 24f\n"
-      "st1b { z14.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z14.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "24:"  // Store to output array: Accumulator row 2 oddments: End
-      "subs x24, x24, x21\n"
+      "subs x25, x25, x22\n"
       "beq 28f\n"
-      "whilelt p0.s, x9, x28\n"
-      "cmp x24, x22\n"
-      "csel x19, x24, x22, LT\n"
-      "lsr x20, x19, #0x2\n"
+      "whilelt p0.s, x10, x9\n"
+      "cmp x25, x23\n"
+      "csel x20, x25, x23, LT\n"
+      "lsr x21, x20, #0x2\n"
       "mov x12, #0x0\n"
-      "and x19, x19, #0x3\n"
-      "cbz x20, 26f\n"
+      "and x20, x20, #0x3\n"
+      "cbz x21, 26f\n"
       "25:"  // Store to output array: Accumulator row 3 loop
       ".inst 0xc0860474  // mova { z20.s-z23.s }, za3h.s[x12]\n"
       ".inst 0xc1a8ac14  // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z8.s\n"
       "add x12, x12, #0x4\n"
       ".inst 0xc1a7aa34  // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n"
-      "cmp x12, x20, LSL #2\n"
+      "cmp x12, x21, LSL #2\n"
       ".inst 0xc1a6ab14  // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n"
       ".inst 0xc1a4ccb4  // sclamp { z20.s-z23.s }, z5.s, z4.s\n"
-      "st1b { z20.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z21.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z22.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
-      "st1b { z23.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z20.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z21.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z22.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
+      "st1b { z23.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "blt 25b\n"
       "26:"  // Store to output array: Accumulator row 3 oddments
-      "cbz x19, 27f\n"
+      "cbz x20, 27f\n"
       ".inst 0xc0860460  // mova { z0.s-z3.s }, za3h.s[x12]\n"
       ".inst 0xc1a8ac00  // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z8.s\n"
-      "subs x19, x19, #0x1\n"
+      "subs x20, x20, #0x1\n"
       ".inst 0xc1a7aa20  // srshl { z0.s-z3.s }, { z0.s-z3.s }, z7.s\n"
       ".inst 0xc1a6ab00  // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n"
       ".inst 0xc1a4cca0  // sclamp { z0.s-z3.s }, z5.s, z4.s\n"
-      "st1b { z0.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "st1b { z0.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "subs x19, x19, #0x1\n"
-      "st1b { z1.s }, p0, [x25]\n"
-      "add x25, x25, x23\n"
+      "subs x20, x20, #0x1\n"
+      "st1b { z1.s }, p0, [x26]\n"
+      "add x26, x26, x24\n"
       "beq 27f\n"
-      "st1b { z2.s }, p0, [x25]\n"
+      "st1b { z2.s }, p0, [x26]\n"
       "27:"  // Store to output array: Accumulator row 3 oddments: End
       "28:"  // Store to output array: End
-      "tbz x15, #0, 30f\n"
+      "tbz x16, #0, 30f\n"
       "mov x12, #0x0\n"
-      "cntw x19\n"
+      "cntw x20\n"
       "29:"  // Store to output array: Refill accumulators: Loop
-      ".inst 0xa040c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n"
+      ".inst 0xa040c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n"
       ".inst 0xc0840480  // mova za0h.s[x12], { z4.s-z7.s }\n"
-      ".inst 0xa041c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x4, MUL VL]\n"
+      ".inst 0xa041c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n"
       ".inst 0xc0840601  // mova za1h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa042c1d0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x14, #0x8, MUL VL]\n"
+      ".inst 0xa042c1f0  // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n"
       ".inst 0xc0840602  // mova za2h.s[x12], { z16.s-z19.s }\n"
-      ".inst 0xa043c1c4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x14, #0xc, MUL VL]\n"
+      ".inst 0xa043c1e4  // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n"
       ".inst 0xc0840483  // mova za3h.s[x12], { z4.s-z7.s }\n"
       "add x12, x12, #0x4\n"
-      "cmp x12, x19\n"
-      "addvl x14, x14, #16\n"
+      "cmp x12, x20\n"
+      "addvl x15, x15, #16\n"
       "blt 29b\n"
       "30:"  // End block
-      "incw x9\n"
-      "cmp x9, x28\n"
+      "incw x10\n"
+      "cmp x10, x9\n"
       "blt 3b\n"
-      "incw x10, ALL, MUL #4\n"
-      "cmp x10, x11\n"
-      "mov x9, #0x0\n"
-      "mov x27, x26\n"
+      "incw x11, ALL, MUL #4\n"
+      "cmp x11, x13\n"
+      "mov x10, #0x0\n"
+      "mov x28, x27\n"
       "blt 3b\n"
       ".inst 0xd503467f  // SMSTOP\n"
       :
       : [args] "r" (&args), [offsetof_A] "I" (offsetof(KernelArgs, A)), [offsetof_B] "I" (offsetof(KernelArgs, B)), [offsetof_C] "I" (offsetof(KernelArgs, C)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_M] "I" (offsetof(KernelArgs, M)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_Requantize32_c_offset] "I" (offsetof(Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [offsetof_accumulator_buffer] "I" (offsetof(KernelArgs, accumulator_buffer)), [offsetof_bias] "I" (offsetof(KernelArgs, bias)), [offsetof_flags] "I" (offsetof(KernelArgs, flags)), [offsetof_kstride_bytes] "I" (offsetof(KernelArgs, kstride_bytes)), [offsetof_ldcb] "I" (offsetof(KernelArgs, ldcb)), [offsetof_n_0] "I" (offsetof(KernelArgs, n_0)), [rq] "r" (&rq)
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp
index c0b6b30762..13f2e488dd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -106,63 +106,63 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
@@ -180,130 +180,130 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "addvl x12, x12, #4\n"
       "addvl x11, x11, #4\n"
       "addvl x10, x10, #4\n"
       "addvl x9, x9, #4\n"
-      "addvl x28, x28, #4\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       "ble 12f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       "12:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 7b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
       "uzp1 z9.d, z9.d, z13.d\n"
       "uzp1 z10.d, z10.d, z14.d\n"
       "uzp1 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 13f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -313,82 +313,82 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z10.s, p5/M, z10.s, z0.s\n"
       "fmax z11.s, p5/M, z11.s, z0.s\n"
       "13:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
       "14:"  // Height 1: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 86f\n"
       "15:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 17f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "17:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 18f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 18f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 20f\n"
       "18:"  // Height 2: no bias
       "tbz %x[flags], #0, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
@@ -404,131 +404,131 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "20:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "21:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 22f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 23f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 23f\n"
       "22:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "23:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 25f\n"
       "24:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
+      "addvl x12, x12, #4\n"
       "addvl x11, x11, #4\n"
       "addvl x10, x10, #4\n"
       "addvl x9, x9, #4\n"
-      "addvl x28, x28, #4\n"
       "bgt 24b\n"
       "25:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       "ble 26f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       "26:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 21b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x12, x19, LSL #2\n"
+      "add x25, x13, x20, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
@@ -536,10 +536,10 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z7.s, p5/M, z7.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -557,67 +557,67 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z10.s, p5/M, z10.s, z0.s\n"
       "fmax z11.s, p5/M, z11.s, z0.s\n"
       "27:"  // Height 2: No activation
-      "st1w { z7.s }, p4, [x12]\n"
-      "st1w { z12.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x13]\n"
+      "st1w { z12.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
       "28:"  // Height 2: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 16b\n"
       "b 86f\n"
       "29:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 31f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "31:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 32f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 32f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -631,29 +631,29 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "b 34f\n"
       "32:"  // Height 3: no bias
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
@@ -681,176 +681,176 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "34:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "35:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 37f\n"
       "36:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "37:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 39f\n"
       "38:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "bgt 38b\n"
       "39:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
+      "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ble 40f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x28, x28, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "40:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
@@ -860,10 +860,10 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z18.d, z18.d, z22.d\n"
       "uzp1 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 41f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z7.s, p5/M, z7.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -889,71 +889,71 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z18.s, p5/M, z18.s, z0.s\n"
       "fmax z19.s, p5/M, z19.s, z0.s\n"
       "41:"  // Height 3: No activation
-      "st1w { z7.s }, p4, [x12]\n"
-      "st1w { z12.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x13]\n"
+      "st1w { z12.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "42:"  // Height 3: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 30b\n"
       "b 86f\n"
       "43:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 45f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "45:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 46f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 46f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -967,38 +967,38 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "b 48f\n"
       "46:"  // Height 4: no bias
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
@@ -1022,182 +1022,182 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "48:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 51f\n"
       "50:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "51:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 53f\n"
       "52:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "bgt 52b\n"
       "53:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
+      "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ble 54f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x28, x28, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "54:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
@@ -1212,10 +1212,10 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z22.d, z19.d, z23.d\n"
       "uzp2 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 55f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z7.s, p5/M, z7.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -1249,75 +1249,75 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z18.s, p5/M, z18.s, z0.s\n"
       "fmax z19.s, p5/M, z19.s, z0.s\n"
       "55:"  // Height 4: No activation
-      "st1w { z7.s }, p4, [x12]\n"
-      "st1w { z12.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x13]\n"
+      "st1w { z12.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "56:"  // Height 4: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 44b\n"
       "b 86f\n"
       "57:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "58:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 59f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 59f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 59f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "59:"  // Height 5: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 60f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 60f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -1339,47 +1339,47 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "b 62f\n"
       "60:"  // Height 5: no bias
       "tbz %x[flags], #0, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
       "zip2 z23.d, z24.d, z23.d\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
@@ -1415,119 +1415,119 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "62:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "63:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 64f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 65f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 65f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 65f\n"
       "64:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "65:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 67f\n"
       "66:"  // Height 5: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "sub x26, x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "cmp x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       ".inst 0x6467e49a  // bfmmla z26.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
-      "addvl x11, x11, #4\n"
+      "addvl x12, x12, #4\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "addvl x10, x10, #4\n"
+      "addvl x11, x11, #4\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
-      "addvl x28, x28, #4\n"
+      "addvl x9, x9, #4\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6467e4bb  // bfmmla z27.s, z5.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
@@ -1535,86 +1535,86 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "bgt 66b\n"
       "67:"  // Height 5: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "addvl x11, x11, #2\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "addvl x10, x10, #2\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "addvl x9, x9, #2\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       ".inst 0x6467e49a  // bfmmla z26.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
-      "addvl x28, x28, #2\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
       "ble 68f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x28, x28, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6467e4bb  // bfmmla z27.s, z5.h, z7.h\n"
@@ -1622,18 +1622,18 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "68:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 63b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
@@ -1652,10 +1652,10 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z26.d, z26.d, z30.d\n"
       "uzp1 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 69f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z7.s, p5/M, z7.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -1697,82 +1697,82 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z26.s, p5/M, z26.s, z0.s\n"
       "fmax z27.s, p5/M, z27.s, z0.s\n"
       "69:"  // Height 5: No activation
-      "st1w { z7.s }, p4, [x12]\n"
-      "st1w { z12.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x13]\n"
+      "st1w { z12.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "70:"  // Height 5: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 58b\n"
       "b 86f\n"
       "71:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x18\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "72:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 73f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 73f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 73f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "73:"  // Height 6: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 74f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 74f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -1794,55 +1794,55 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "b 76f\n"
       "74:"  // Height 6: no bias
       "tbz %x[flags], #0, 75f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip2 z15.d, z16.d, z15.d\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip2 z21.d, z18.d, z21.d\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
       "zip2 z23.d, z24.d, z23.d\n"
       "zip1 z24.d, z25.d, z28.d\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z29.d, z26.d, z29.d\n"
       "zip1 z26.d, z27.d, z30.d\n"
       "zip2 z30.d, z27.d, z30.d\n"
@@ -1875,124 +1875,124 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "76:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "77:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 78f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 79f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 79f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 79f\n"
       "78:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "79:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 81f\n"
       "80:"  // Height 6: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      "ld1rqh { z6.h }, p0/Z, [x20]\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
+      "ld1rqh { z6.h }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "sub x26, x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "cmp x26, #0x8\n"
-      "add x25, x25, #0x10\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "add x20, x20, #0x10\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       ".inst 0x6467e49a  // bfmmla z26.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
-      "addvl x11, x11, #4\n"
+      "addvl x12, x12, #4\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "addvl x10, x10, #4\n"
+      "addvl x11, x11, #4\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
-      "addvl x28, x28, #4\n"
+      "addvl x9, x9, #4\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6467e4bb  // bfmmla z27.s, z5.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
@@ -2000,87 +2000,87 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "bgt 80b\n"
       "81:"  // Height 6: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      "ld1rqh { z6.h }, p0/Z, [x20]\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
+      "ld1rqh { z6.h }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "subs x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "addvl x11, x11, #2\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "addvl x10, x10, #2\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "addvl x9, x9, #2\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       ".inst 0x6467e49a  // bfmmla z26.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
-      "addvl x28, x28, #2\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
       "ble 82f\n"
-      "ld1h { z7.h }, p5/Z, [x11]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x12]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x28, #1, MUL VL]\n"
-      "addvl x28, x28, #2\n"
+      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6467e473  // bfmmla z19.s, z3.h, z7.h\n"
       ".inst 0x6467e4bb  // bfmmla z27.s, z5.h, z7.h\n"
@@ -2088,21 +2088,21 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "82:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 77b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
@@ -2123,10 +2123,10 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z30.d, z27.d, z31.d\n"
       "uzp2 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 83f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z7.s, p5/M, z7.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -2176,50 +2176,50 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL (
       "fmax z26.s, p5/M, z26.s, z0.s\n"
       "fmax z27.s, p5/M, z27.s, z0.s\n"
       "83:"  // Height 6: No activation
-      "st1w { z7.s }, p4, [x12]\n"
-      "st1w { z12.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z23.s }, p4, [x21]\n"
-      "st1w { z28.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z29.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z30.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x13]\n"
+      "st1w { z12.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z23.s }, p4, [x22]\n"
+      "st1w { z28.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z29.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z30.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "84:"  // Height 6: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 72b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 86f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 85f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "85:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "86:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp
index 181022bf51..5f093bf08a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,53 +105,53 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
-      "addvl x14, x14, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
+      "addvl x15, x15, #4\n"
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
       "b 6f\n"
       "5:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -159,64 +159,64 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "bne 7b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -226,74 +226,74 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z10.h, p4/M, z10.h, z0.h\n"
       "fmax z11.h, p4/M, z11.h, z0.h\n"
       "12:"  // Height 1: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
       "13:"  // Height 1: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 16f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 16f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 16f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "16:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 17f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 17f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "b 19f\n"
       "17:"  // Height 2: no bias
       "tbz %x[flags], #0, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
       "b 19f\n"
       "18:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -305,80 +305,80 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "19:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "20:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 21f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 22f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 22f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 22f\n"
       "21:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "22:"  // Height 2: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 24f\n"
       "23:"  // Height 2: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x10, x10, #1\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x11, x11, #1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
-      "add x24, x24, #0x2\n"
+      "add x25, x25, #0x2\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
+      "addvl x10, x10, #1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
       "addvl x9, x9, #1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "addvl x28, x28, #1\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 23b\n"
       "24:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "bne 20b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -396,87 +396,87 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z14.h, p4/M, z14.h, z0.h\n"
       "fmax z15.h, p4/M, z15.h, z0.h\n"
       "25:"  // Height 2: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 29f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 29f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 29f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "29:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 30f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 30f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 32f\n"
       "30:"  // Height 3: no bias
       "tbz %x[flags], #0, 31f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
       "b 32f\n"
       "31:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -492,95 +492,95 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "32:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "33:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 34f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 35f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 35f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 35f\n"
       "34:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "35:"  // Height 3: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 37f\n"
       "36:"  // Height 3: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "add x25, x25, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "add x26, x26, #0x2\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x2\n"
       "add x24, x24, #0x2\n"
-      "add x23, x23, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 36b\n"
       "37:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "cmp x28, x20\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "bne 33b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -606,71 +606,71 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z18.h, p4/M, z18.h, z0.h\n"
       "fmax z19.h, p4/M, z19.h, z0.h\n"
       "38:"  // Height 3: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 42f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 42f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 42f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "42:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 43f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 43f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -680,26 +680,26 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "b 45f\n"
       "43:"  // Height 4: no bias
       "tbz %x[flags], #0, 44f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
       "b 45f\n"
       "44:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -719,94 +719,94 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "46:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "48:"  // Height 4: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 50f\n"
       "49:"  // Height 4: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "add x25, x25, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "add x26, x26, #0x2\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "subs x26, x26, #0x1\n"
-      "add x24, x24, #0x2\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x2\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "add x24, x24, #0x2\n"
       "add x23, x23, #0x2\n"
-      "add x22, x22, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
-      "addvl x28, x28, #1\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
+      "addvl x9, x9, #1\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 49b\n"
       "50:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "cmp x28, x20\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
@@ -814,15 +814,15 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "bne 46b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -856,75 +856,75 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z22.h, p4/M, z22.h, z0.h\n"
       "fmax z23.h, p4/M, z23.h, z0.h\n"
       "51:"  // Height 4: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 55f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 55f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 55f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "55:"  // Height 5: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 56f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 56f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -938,31 +938,31 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "b 58f\n"
       "56:"  // Height 5: no bias
       "tbz %x[flags], #0, 57f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p3/Z, [x21]\n"
-      "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p3/Z, [x22]\n"
+      "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n"
       "b 58f\n"
       "57:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -986,104 +986,104 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "58:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "59:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 60f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 61f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 61f\n"
       "60:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "61:"  // Height 5: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 63f\n"
       "62:"  // Height 5: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "add x24, x24, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "add x25, x25, #0x2\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
+      "add x24, x24, #0x2\n"
       "add x23, x23, #0x2\n"
-      "add x22, x22, #0x2\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "add x21, x21, #0x2\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "add x22, x22, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
       "fmla z26.h, p4/M, z6.h, z4.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 62b\n"
       "63:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "cmp x27, x19\n"
-      "addvl x11, x11, #1\n"
+      "cmp x28, x20\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -1095,16 +1095,16 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "bne 59b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -1146,82 +1146,82 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z26.h, p4/M, z26.h, z0.h\n"
       "fmax z27.h, p4/M, z27.h, z0.h\n"
       "64:"  // Height 5: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p3, [x21]\n"
-      "st1h { z25.h }, p2, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p1, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p0, [x21, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p3, [x22]\n"
+      "st1h { z25.h }, p2, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p1, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p0, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0xc\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0xc\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 68f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 68f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 68f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "68:"  // Height 6: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x13\n"
-      "cbz x14, 69f\n"
-      "ld1h { z8.h }, p4/Z, [x14]\n"
-      "ld1h { z9.h }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x14\n"
+      "cbz x15, 69f\n"
+      "ld1h { z8.h }, p4/Z, [x15]\n"
+      "ld1h { z9.h }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1239,36 +1239,36 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "b 71f\n"
       "69:"  // Height 6: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z9.h }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x12, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #1\n"
-      "ld1h { z11.h }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p3/Z, [x21]\n"
-      "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1h { z28.h }, p3/Z, [x20]\n"
-      "ld1h { z29.h }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z30.h }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z31.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #1\n"
+      "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p3/Z, [x22]\n"
+      "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1h { z28.h }, p3/Z, [x21]\n"
+      "ld1h { z29.h }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z30.h }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z31.h }, p0/Z, [x21, #3, MUL VL]\n"
       "b 71f\n"
       "70:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1296,116 +1296,116 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "71:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "72:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 74f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 74f\n"
       "73:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "74:"  // Height 6: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1rh { z5.h }, p4/Z, [x20]\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1rh { z5.h }, p4/Z, [x21]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "ble 76f\n"
       "75:"  // Height 6: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z28.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "add x24, x24, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "add x25, x25, #0x2\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
+      "add x24, x24, #0x2\n"
       "add x23, x23, #0x2\n"
-      "add x22, x22, #0x2\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
+      "add x22, x22, #0x2\n"
       "add x21, x21, #0x2\n"
-      "add x20, x20, #0x2\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
       "fmla z29.h, p4/M, z7.h, z5.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
       "fmla z26.h, p4/M, z6.h, z4.h\n"
       "fmla z30.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x11]\n"
+      "ld1h { z6.h }, p4/Z, [x12]\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "fmla z31.h, p4/M, z7.h, z5.h\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1rh { z5.h }, p4/Z, [x20]\n"
-      "ld1h { z7.h }, p4/Z, [x10]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1rh { z5.h }, p4/Z, [x21]\n"
+      "ld1h { z7.h }, p4/Z, [x11]\n"
       "bgt 75b\n"
       "76:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "cmp x27, x19\n"
-      "addvl x11, x11, #1\n"
+      "cmp x28, x20\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z28.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
       "fmla z29.h, p4/M, z7.h, z5.h\n"
-      "ld1h { z7.h }, p4/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p4/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -1419,17 +1419,17 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "fmla z31.h, p4/M, z7.h, z5.h\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -1479,50 +1479,50 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx (
       "fmax z30.h, p4/M, z30.h, z0.h\n"
       "fmax z31.h, p4/M, z31.h, z0.h\n"
       "77:"  // Height 6: No activation
-      "st1h { z8.h }, p3, [x12]\n"
-      "st1h { z9.h }, p2, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p3, [x21]\n"
-      "st1h { z25.h }, p2, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p1, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p0, [x21, #3, MUL VL]\n"
-      "st1h { z28.h }, p3, [x20]\n"
-      "st1h { z29.h }, p2, [x20, #1, MUL VL]\n"
-      "st1h { z30.h }, p1, [x20, #2, MUL VL]\n"
-      "st1h { z31.h }, p0, [x20, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x13]\n"
+      "st1h { z9.h }, p2, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p3, [x22]\n"
+      "st1h { z25.h }, p2, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p1, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p0, [x22, #3, MUL VL]\n"
+      "st1h { z28.h }, p3, [x21]\n"
+      "st1h { z29.h }, p2, [x21, #1, MUL VL]\n"
+      "st1h { z30.h }, p1, [x21, #2, MUL VL]\n"
+      "st1h { z31.h }, p0, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp
index 0f995812d8..0b543b667f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,53 +105,53 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
-      "addvl x14, x14, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
+      "addvl x15, x15, #4\n"
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
       "b 6f\n"
       "5:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -159,222 +159,222 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "cmp x27, #0x8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "addvl x12, x12, #8\n"
       "addvl x11, x11, #8\n"
       "addvl x10, x10, #8\n"
       "addvl x9, x9, #8\n"
-      "addvl x28, x28, #8\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "12:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 7b\n"
       "tbz %x[flags], #1, 13f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -384,74 +384,74 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z10.h, p5/M, z10.h, z0.h\n"
       "fmax z11.h, p5/M, z11.h, z0.h\n"
       "13:"  // Height 1: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
       "14:"  // Height 1: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 86f\n"
       "15:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 17f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 17f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 17f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "17:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 18f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 18f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "b 20f\n"
       "18:"  // Height 2: no bias
       "tbz %x[flags], #0, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
       "b 20f\n"
       "19:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -463,294 +463,294 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "20:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "21:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 22f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 23f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 23f\n"
       "22:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "23:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 25f\n"
       "24:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
+      "cmp x27, #0x8\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
-      "addvl x11, x11, #8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
+      "addvl x12, x12, #8\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "addvl x11, x11, #8\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
       "fmla z15.h, z7.h, z1.h[7]\n"
       "bgt 24b\n"
       "25:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
       "fmla z15.h, z7.h, z1.h[7]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "26:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 21b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -768,87 +768,87 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z14.h, p5/M, z14.h, z0.h\n"
       "fmax z15.h, p5/M, z15.h, z0.h\n"
       "27:"  // Height 2: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
       "28:"  // Height 2: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 16b\n"
       "b 86f\n"
       "29:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 31f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 31f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 31f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "31:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 32f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 32f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 34f\n"
       "32:"  // Height 3: no bias
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
       "b 34f\n"
       "33:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -864,166 +864,166 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "34:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "35:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 37f\n"
       "36:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "37:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 39f\n"
       "38:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "cmp x27, #0x8\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
-      "addvl x11, x11, #8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
+      "addvl x12, x12, #8\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "addvl x11, x11, #8\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -1032,197 +1032,197 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[7]\n"
       "bgt 38b\n"
       "39:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "ble 40f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x11, x11, #1\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
       "fmla z15.h, z7.h, z1.h[7]\n"
       "fmla z19.h, z7.h, z2.h[7]\n"
       "40:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "tbz %x[flags], #1, 41f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -1248,71 +1248,71 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z18.h, p5/M, z18.h, z0.h\n"
       "fmax z19.h, p5/M, z19.h, z0.h\n"
       "41:"  // Height 3: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
       "42:"  // Height 3: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 30b\n"
       "b 86f\n"
       "43:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 45f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 45f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 45f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "45:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 46f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 46f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1322,26 +1322,26 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "b 48f\n"
       "46:"  // Height 4: no bias
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
       "b 48f\n"
       "47:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -1361,201 +1361,201 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "48:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 51f\n"
       "50:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "51:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 53f\n"
       "52:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "cmp x26, #0x8\n"
-      "add x25, x25, #0x10\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z22.h, z6.h, z3.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "fmla z23.h, z7.h, z3.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z22.h, z6.h, z3.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "fmla z23.h, z7.h, z3.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z22.h, z6.h, z3.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "fmla z23.h, z7.h, z3.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z22.h, z6.h, z3.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "fmla z23.h, z7.h, z3.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z22.h, z6.h, z3.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "fmla z23.h, z7.h, z3.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z22.h, z6.h, z3.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "fmla z23.h, z7.h, z3.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z22.h, z6.h, z3.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
-      "addvl x11, x11, #8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
+      "addvl x12, x12, #8\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "fmla z23.h, z7.h, z3.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "addvl x11, x11, #8\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -1566,28 +1566,28 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[7]\n"
       "bgt 52b\n"
       "53:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
@@ -1597,25 +1597,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[0]\n"
       "fmla z23.h, z7.h, z3.h[0]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z22.h, z6.h, z3.h[1]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
@@ -1623,25 +1623,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[1]\n"
       "fmla z23.h, z7.h, z3.h[1]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z22.h, z6.h, z3.h[2]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
@@ -1649,25 +1649,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[2]\n"
       "fmla z23.h, z7.h, z3.h[2]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z22.h, z6.h, z3.h[3]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
@@ -1675,25 +1675,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[3]\n"
       "fmla z23.h, z7.h, z3.h[3]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z22.h, z6.h, z3.h[4]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
@@ -1701,25 +1701,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[4]\n"
       "fmla z23.h, z7.h, z3.h[4]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z22.h, z6.h, z3.h[5]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
@@ -1727,25 +1727,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[5]\n"
       "fmla z23.h, z7.h, z3.h[5]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z22.h, z6.h, z3.h[6]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
@@ -1753,22 +1753,22 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[6]\n"
       "fmla z23.h, z7.h, z3.h[6]\n"
       "ble 54f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -1778,19 +1778,19 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[7]\n"
       "fmla z23.h, z7.h, z3.h[7]\n"
       "54:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "tbz %x[flags], #1, 55f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -1824,75 +1824,75 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z22.h, p5/M, z22.h, z0.h\n"
       "fmax z23.h, p5/M, z23.h, z0.h\n"
       "55:"  // Height 4: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
       "56:"  // Height 4: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 44b\n"
       "b 86f\n"
       "57:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "58:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 59f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 59f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 59f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "59:"  // Height 5: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 60f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 60f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1906,31 +1906,31 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "b 62f\n"
       "60:"  // Height 5: no bias
       "tbz %x[flags], #0, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p4/Z, [x21]\n"
-      "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p4/Z, [x22]\n"
+      "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
       "b 62f\n"
       "61:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -1954,236 +1954,236 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "62:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "63:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 64f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 65f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 65f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 65f\n"
       "64:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "65:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 67f\n"
       "66:"  // Height 5: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "cmp x26, #0x8\n"
-      "add x25, x25, #0x10\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "add x24, x24, #0x10\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z22.h, z6.h, z3.h[0]\n"
       "fmla z26.h, z6.h, z4.h[0]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "fmla z23.h, z7.h, z3.h[0]\n"
       "fmla z27.h, z7.h, z4.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
       "fmla z25.h, z7.h, z4.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z22.h, z6.h, z3.h[1]\n"
       "fmla z26.h, z6.h, z4.h[1]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "fmla z23.h, z7.h, z3.h[1]\n"
       "fmla z27.h, z7.h, z4.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
       "fmla z25.h, z7.h, z4.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z22.h, z6.h, z3.h[2]\n"
       "fmla z26.h, z6.h, z4.h[2]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "fmla z23.h, z7.h, z3.h[2]\n"
       "fmla z27.h, z7.h, z4.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
       "fmla z25.h, z7.h, z4.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z22.h, z6.h, z3.h[3]\n"
       "fmla z26.h, z6.h, z4.h[3]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "fmla z23.h, z7.h, z3.h[3]\n"
       "fmla z27.h, z7.h, z4.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
       "fmla z25.h, z7.h, z4.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z22.h, z6.h, z3.h[4]\n"
       "fmla z26.h, z6.h, z4.h[4]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "fmla z23.h, z7.h, z3.h[4]\n"
       "fmla z27.h, z7.h, z4.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
       "fmla z25.h, z7.h, z4.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z22.h, z6.h, z3.h[5]\n"
       "fmla z26.h, z6.h, z4.h[5]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "fmla z23.h, z7.h, z3.h[5]\n"
       "fmla z27.h, z7.h, z4.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
       "fmla z25.h, z7.h, z4.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z22.h, z6.h, z3.h[6]\n"
       "fmla z26.h, z6.h, z4.h[6]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
-      "addvl x11, x11, #8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
+      "addvl x12, x12, #8\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "fmla z23.h, z7.h, z3.h[6]\n"
       "fmla z27.h, z7.h, z4.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "addvl x11, x11, #8\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
       "fmla z25.h, z7.h, z4.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -2196,31 +2196,31 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[7]\n"
       "bgt 66b\n"
       "67:"  // Height 5: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
@@ -2232,25 +2232,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[0]\n"
       "fmla z27.h, z7.h, z4.h[0]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
       "fmla z25.h, z7.h, z4.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
@@ -2262,25 +2262,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[1]\n"
       "fmla z27.h, z7.h, z4.h[1]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
       "fmla z25.h, z7.h, z4.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
@@ -2292,25 +2292,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[2]\n"
       "fmla z27.h, z7.h, z4.h[2]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
       "fmla z25.h, z7.h, z4.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
@@ -2322,25 +2322,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[3]\n"
       "fmla z27.h, z7.h, z4.h[3]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
       "fmla z25.h, z7.h, z4.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
@@ -2352,25 +2352,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[4]\n"
       "fmla z27.h, z7.h, z4.h[4]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
       "fmla z25.h, z7.h, z4.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
@@ -2382,25 +2382,25 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[5]\n"
       "fmla z27.h, z7.h, z4.h[5]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
       "fmla z25.h, z7.h, z4.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
@@ -2412,24 +2412,24 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[6]\n"
       "fmla z27.h, z7.h, z4.h[6]\n"
       "ble 68f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
       "fmla z25.h, z7.h, z4.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -2441,20 +2441,20 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[7]\n"
       "fmla z27.h, z7.h, z4.h[7]\n"
       "68:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 63b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "tbz %x[flags], #1, 69f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -2496,82 +2496,82 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z26.h, p5/M, z26.h, z0.h\n"
       "fmax z27.h, p5/M, z27.h, z0.h\n"
       "69:"  // Height 5: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p4, [x21]\n"
-      "st1h { z25.h }, p3, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p2, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p1, [x21, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p4, [x22]\n"
+      "st1h { z25.h }, p3, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p2, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p1, [x22, #3, MUL VL]\n"
       "70:"  // Height 5: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 58b\n"
       "b 86f\n"
       "71:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0xc\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0xc\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "72:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "cnth x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "add x19, x28, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "cnth x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "add x20, x9, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 73f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 73f\n"
-      "dech x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "dech x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 73f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "73:"  // Height 6: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x13\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x13\n"
-      "cbz x14, 74f\n"
-      "ld1h { z8.h }, p5/Z, [x14]\n"
-      "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x14\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x14\n"
+      "cbz x15, 74f\n"
+      "ld1h { z8.h }, p5/Z, [x15]\n"
+      "ld1h { z9.h }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -2589,36 +2589,36 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "b 76f\n"
       "74:"  // Height 6: no bias
       "tbz %x[flags], #0, 75f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #1\n"
-      "ld1h { z11.h }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p4/Z, [x21]\n"
-      "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1h { z28.h }, p4/Z, [x20]\n"
-      "ld1h { z29.h }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z30.h }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z31.h }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #1\n"
+      "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p4/Z, [x22]\n"
+      "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1h { z28.h }, p4/Z, [x21]\n"
+      "ld1h { z29.h }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z30.h }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z31.h }, p1/Z, [x21, #3, MUL VL]\n"
       "b 76f\n"
       "75:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -2646,271 +2646,271 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "76:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "77:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 78f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 79f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 79f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 79f\n"
       "78:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "79:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 81f\n"
       "80:"  // Height 6: Multiply loop: Main loop head
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "cmp x26, #0x8\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
       "add x25, x25, #0x10\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z28.h, z6.h, z5.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "add x20, x20, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "add x21, x21, #0x10\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
       "fmla z29.h, z7.h, z5.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z22.h, z6.h, z3.h[0]\n"
       "fmla z26.h, z6.h, z4.h[0]\n"
       "fmla z30.h, z6.h, z5.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "fmla z23.h, z7.h, z3.h[0]\n"
       "fmla z27.h, z7.h, z4.h[0]\n"
       "fmla z31.h, z7.h, z5.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
       "fmla z28.h, z6.h, z5.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
       "fmla z25.h, z7.h, z4.h[1]\n"
       "fmla z29.h, z7.h, z5.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z22.h, z6.h, z3.h[1]\n"
       "fmla z26.h, z6.h, z4.h[1]\n"
       "fmla z30.h, z6.h, z5.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "fmla z23.h, z7.h, z3.h[1]\n"
       "fmla z27.h, z7.h, z4.h[1]\n"
       "fmla z31.h, z7.h, z5.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
       "fmla z28.h, z6.h, z5.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
       "fmla z25.h, z7.h, z4.h[2]\n"
       "fmla z29.h, z7.h, z5.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z22.h, z6.h, z3.h[2]\n"
       "fmla z26.h, z6.h, z4.h[2]\n"
       "fmla z30.h, z6.h, z5.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "fmla z23.h, z7.h, z3.h[2]\n"
       "fmla z27.h, z7.h, z4.h[2]\n"
       "fmla z31.h, z7.h, z5.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
       "fmla z28.h, z6.h, z5.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
       "fmla z25.h, z7.h, z4.h[3]\n"
       "fmla z29.h, z7.h, z5.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z22.h, z6.h, z3.h[3]\n"
       "fmla z26.h, z6.h, z4.h[3]\n"
       "fmla z30.h, z6.h, z5.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "fmla z23.h, z7.h, z3.h[3]\n"
       "fmla z27.h, z7.h, z4.h[3]\n"
       "fmla z31.h, z7.h, z5.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
       "fmla z28.h, z6.h, z5.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
       "fmla z25.h, z7.h, z4.h[4]\n"
       "fmla z29.h, z7.h, z5.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z22.h, z6.h, z3.h[4]\n"
       "fmla z26.h, z6.h, z4.h[4]\n"
       "fmla z30.h, z6.h, z5.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "fmla z23.h, z7.h, z3.h[4]\n"
       "fmla z27.h, z7.h, z4.h[4]\n"
       "fmla z31.h, z7.h, z5.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
       "fmla z28.h, z6.h, z5.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
       "fmla z25.h, z7.h, z4.h[5]\n"
       "fmla z29.h, z7.h, z5.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z22.h, z6.h, z3.h[5]\n"
       "fmla z26.h, z6.h, z4.h[5]\n"
       "fmla z30.h, z6.h, z5.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "fmla z23.h, z7.h, z3.h[5]\n"
       "fmla z27.h, z7.h, z4.h[5]\n"
       "fmla z31.h, z7.h, z5.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
       "fmla z28.h, z6.h, z5.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
       "fmla z25.h, z7.h, z4.h[6]\n"
       "fmla z29.h, z7.h, z5.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z22.h, z6.h, z3.h[6]\n"
       "fmla z26.h, z6.h, z4.h[6]\n"
       "fmla z30.h, z6.h, z5.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x11, #7, MUL VL]\n"
-      "addvl x11, x11, #8\n"
+      "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n"
+      "addvl x12, x12, #8\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "fmla z23.h, z7.h, z3.h[6]\n"
       "fmla z27.h, z7.h, z4.h[6]\n"
       "fmla z31.h, z7.h, z5.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n"
+      "addvl x11, x11, #8\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
       "fmla z28.h, z6.h, z5.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
       "fmla z25.h, z7.h, z4.h[7]\n"
       "fmla z29.h, z7.h, z5.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -2925,34 +2925,34 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[7]\n"
       "bgt 80b\n"
       "81:"  // Height 6: Multiply loop: Single iteration only
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z28.h, z6.h, z5.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
       "fmla z29.h, z7.h, z5.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
@@ -2966,27 +2966,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[0]\n"
       "fmla z31.h, z7.h, z5.h[0]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
       "fmla z28.h, z6.h, z5.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
       "fmla z25.h, z7.h, z4.h[1]\n"
       "fmla z29.h, z7.h, z5.h[1]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
@@ -3000,27 +3000,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[1]\n"
       "fmla z31.h, z7.h, z5.h[1]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
       "fmla z28.h, z6.h, z5.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
       "fmla z25.h, z7.h, z4.h[2]\n"
       "fmla z29.h, z7.h, z5.h[2]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
@@ -3034,27 +3034,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[2]\n"
       "fmla z31.h, z7.h, z5.h[2]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
       "fmla z28.h, z6.h, z5.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
       "fmla z25.h, z7.h, z4.h[3]\n"
       "fmla z29.h, z7.h, z5.h[3]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
@@ -3068,27 +3068,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[3]\n"
       "fmla z31.h, z7.h, z5.h[3]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
       "fmla z28.h, z6.h, z5.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
       "fmla z25.h, z7.h, z4.h[4]\n"
       "fmla z29.h, z7.h, z5.h[4]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
@@ -3102,27 +3102,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[4]\n"
       "fmla z31.h, z7.h, z5.h[4]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
       "fmla z28.h, z6.h, z5.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
       "fmla z25.h, z7.h, z4.h[5]\n"
       "fmla z29.h, z7.h, z5.h[5]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
@@ -3136,27 +3136,27 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[5]\n"
       "fmla z31.h, z7.h, z5.h[5]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
       "fmla z28.h, z6.h, z5.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
       "fmla z25.h, z7.h, z4.h[6]\n"
       "fmla z29.h, z7.h, z5.h[6]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
@@ -3170,26 +3170,26 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[6]\n"
       "fmla z31.h, z7.h, z5.h[6]\n"
       "ble 82f\n"
-      "ld1h { z6.h }, p5/Z, [x11]\n"
-      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x12]\n"
+      "ld1h { z7.h }, p5/Z, [x11]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
       "fmla z28.h, z6.h, z5.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
       "fmla z25.h, z7.h, z4.h[7]\n"
       "fmla z29.h, z7.h, z5.h[7]\n"
-      "ld1h { z7.h }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1h { z7.h }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
       "fmla z18.h, z6.h, z2.h[7]\n"
@@ -3203,21 +3203,21 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[7]\n"
       "fmla z31.h, z7.h, z5.h[7]\n"
       "82:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 77b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "tbz %x[flags], #1, 83f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
       "fmin z8.h, p5/M, z8.h, z1.h\n"
       "fmin z9.h, p5/M, z9.h, z1.h\n"
       "fmin z10.h, p5/M, z10.h, z1.h\n"
@@ -3267,50 +3267,50 @@ void sve_ffhybrid_fp16_mla_6x4VL (
       "fmax z30.h, p5/M, z30.h, z0.h\n"
       "fmax z31.h, p5/M, z31.h, z0.h\n"
       "83:"  // Height 6: No activation
-      "st1h { z8.h }, p4, [x12]\n"
-      "st1h { z9.h }, p3, [x12, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x12, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p4, [x21]\n"
-      "st1h { z25.h }, p3, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p2, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p1, [x21, #3, MUL VL]\n"
-      "st1h { z28.h }, p4, [x20]\n"
-      "st1h { z29.h }, p3, [x20, #1, MUL VL]\n"
-      "st1h { z30.h }, p2, [x20, #2, MUL VL]\n"
-      "st1h { z31.h }, p1, [x20, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x13]\n"
+      "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p4, [x22]\n"
+      "st1h { z25.h }, p3, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p2, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p1, [x22, #3, MUL VL]\n"
+      "st1h { z28.h }, p4, [x21]\n"
+      "st1h { z29.h }, p3, [x21, #1, MUL VL]\n"
+      "st1h { z30.h }, p2, [x21, #2, MUL VL]\n"
+      "st1h { z31.h }, p1, [x21, #3, MUL VL]\n"
       "84:"  // Height 6: Writeback done
-      "dech x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "dech x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 72b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 86f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 85f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "85:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "86:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp
index 7dd4e234d5..32fcac3a45 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,53 +105,53 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
-      "addvl x14, x14, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
+      "addvl x15, x15, #4\n"
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
       "b 6f\n"
       "5:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -159,64 +159,64 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "bne 7b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -226,74 +226,74 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z10.s, p4/M, z10.s, z0.s\n"
       "fmax z11.s, p4/M, z11.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
       "13:"  // Height 1: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 16f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 16f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 16f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "16:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 17f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 17f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "b 19f\n"
       "17:"  // Height 2: no bias
       "tbz %x[flags], #0, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
       "b 19f\n"
       "18:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -305,80 +305,80 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "19:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "20:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 21f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 22f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 22f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 22f\n"
       "21:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "22:"  // Height 2: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 24f\n"
       "23:"  // Height 2: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x10, x10, #1\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x11, x11, #1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
-      "add x24, x24, #0x4\n"
+      "add x25, x25, #0x4\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
+      "addvl x10, x10, #1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "addvl x9, x9, #1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "addvl x28, x28, #1\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 23b\n"
       "24:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "bne 20b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -396,87 +396,87 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z14.s, p4/M, z14.s, z0.s\n"
       "fmax z15.s, p4/M, z15.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 29f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 29f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 29f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "29:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 30f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 30f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 32f\n"
       "30:"  // Height 3: no bias
       "tbz %x[flags], #0, 31f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
       "b 32f\n"
       "31:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -492,95 +492,95 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "32:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "33:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 34f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 35f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 35f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 35f\n"
       "34:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "35:"  // Height 3: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 37f\n"
       "36:"  // Height 3: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "add x25, x25, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "add x26, x26, #0x4\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 36b\n"
       "37:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "cmp x28, x20\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "bne 33b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -606,71 +606,71 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z18.s, p4/M, z18.s, z0.s\n"
       "fmax z19.s, p4/M, z19.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 42f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 42f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 42f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "42:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 43f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 43f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -680,26 +680,26 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "b 45f\n"
       "43:"  // Height 4: no bias
       "tbz %x[flags], #0, 44f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
       "b 45f\n"
       "44:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -719,94 +719,94 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "46:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "48:"  // Height 4: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 50f\n"
       "49:"  // Height 4: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "add x25, x25, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "add x26, x26, #0x4\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "subs x26, x26, #0x1\n"
-      "add x24, x24, #0x4\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x4\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
-      "addvl x28, x28, #1\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
+      "addvl x9, x9, #1\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 49b\n"
       "50:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "cmp x28, x20\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
@@ -814,15 +814,15 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "bne 46b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -856,75 +856,75 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z22.s, p4/M, z22.s, z0.s\n"
       "fmax z23.s, p4/M, z23.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 55f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 55f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 55f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "55:"  // Height 5: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 56f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 56f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -938,31 +938,31 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "b 58f\n"
       "56:"  // Height 5: no bias
       "tbz %x[flags], #0, 57f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n"
       "b 58f\n"
       "57:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -986,104 +986,104 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "58:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "59:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 60f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 61f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 61f\n"
       "60:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "61:"  // Height 5: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 63f\n"
       "62:"  // Height 5: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "add x24, x24, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "add x25, x25, #0x4\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "add x21, x21, #0x4\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "add x22, x22, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
       "fmla z26.s, p4/M, z6.s, z4.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 62b\n"
       "63:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "cmp x27, x19\n"
-      "addvl x11, x11, #1\n"
+      "cmp x28, x20\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -1095,16 +1095,16 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "bne 59b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -1146,82 +1146,82 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z26.s, p4/M, z26.s, z0.s\n"
       "fmax z27.s, p4/M, z27.s, z0.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x21]\n"
-      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x22]\n"
+      "st1w { z25.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x18\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 68f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 68f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 68f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "68:"  // Height 6: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x13\n"
-      "cbz x14, 69f\n"
-      "ld1w { z8.s }, p4/Z, [x14]\n"
-      "ld1w { z9.s }, p4/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x14\n"
+      "cbz x15, 69f\n"
+      "ld1w { z8.s }, p4/Z, [x15]\n"
+      "ld1w { z9.s }, p4/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p4/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1239,36 +1239,36 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "b 71f\n"
       "69:"  // Height 6: no bias
       "tbz %x[flags], #0, 70f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x12, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z11.s }, p0/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p3/Z, [x20]\n"
-      "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p3/Z, [x21]\n"
+      "ld1w { z29.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z31.s }, p0/Z, [x21, #3, MUL VL]\n"
       "b 71f\n"
       "70:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1296,116 +1296,116 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "71:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "72:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 74f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 74f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 74f\n"
       "73:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "74:"  // Height 6: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "ble 76f\n"
       "75:"  // Height 6: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z28.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "add x24, x24, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "add x25, x25, #0x4\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
+      "add x22, x22, #0x4\n"
       "add x21, x21, #0x4\n"
-      "add x20, x20, #0x4\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
       "fmla z29.s, p4/M, z7.s, z5.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
       "fmla z26.s, p4/M, z6.s, z4.s\n"
       "fmla z30.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x11]\n"
+      "ld1w { z6.s }, p4/Z, [x12]\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "fmla z31.s, p4/M, z7.s, z5.s\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1w { z7.s }, p4/Z, [x10]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1w { z7.s }, p4/Z, [x11]\n"
       "bgt 75b\n"
       "76:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "cmp x27, x19\n"
-      "addvl x11, x11, #1\n"
+      "cmp x28, x20\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z28.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
       "fmla z29.s, p4/M, z7.s, z5.s\n"
-      "ld1w { z7.s }, p4/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p4/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -1419,17 +1419,17 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "fmla z31.s, p4/M, z7.s, z5.s\n"
       "bne 72b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -1479,50 +1479,50 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx (
       "fmax z30.s, p4/M, z30.s, z0.s\n"
       "fmax z31.s, p4/M, z31.s, z0.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z8.s }, p3, [x12]\n"
-      "st1w { z9.s }, p2, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x21]\n"
-      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z28.s }, p3, [x20]\n"
-      "st1w { z29.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z30.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z31.s }, p0, [x20, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x13]\n"
+      "st1w { z9.s }, p2, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x22]\n"
+      "st1w { z25.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z28.s }, p3, [x21]\n"
+      "st1w { z29.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z30.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z31.s }, p0, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp
index 3c7e562c89..eb057e7734 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,53 +105,53 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
-      "addvl x14, x14, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
+      "addvl x15, x15, #4\n"
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
       "b 6f\n"
       "5:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -159,134 +159,134 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "cmp x26, #0x4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "cmp x27, #0x4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "addvl x12, x12, #4\n"
       "addvl x11, x11, #4\n"
       "addvl x10, x10, #4\n"
       "addvl x9, x9, #4\n"
-      "addvl x28, x28, #4\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 12f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "12:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 7b\n"
       "tbz %x[flags], #1, 13f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -296,74 +296,74 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z10.s, p5/M, z10.s, z0.s\n"
       "fmax z11.s, p5/M, z11.s, z0.s\n"
       "13:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
       "14:"  // Height 1: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 86f\n"
       "15:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 17f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "17:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 18f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 18f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "b 20f\n"
       "18:"  // Height 2: no bias
       "tbz %x[flags], #0, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "b 20f\n"
       "19:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -375,174 +375,174 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "20:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "21:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 22f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 23f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 23f\n"
       "22:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "23:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 25f\n"
       "24:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "cmp x26, #0x4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
+      "cmp x27, #0x4\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "add x24, x24, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
       "fmla z15.s, z7.s, z1.s[3]\n"
       "bgt 24b\n"
       "25:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "ble 26f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
       "fmla z15.s, z7.s, z1.s[3]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "26:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 21b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -560,87 +560,87 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z14.s, p5/M, z14.s, z0.s\n"
       "fmax z15.s, p5/M, z15.s, z0.s\n"
       "27:"  // Height 2: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
       "28:"  // Height 2: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 16b\n"
       "b 86f\n"
       "29:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 31f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "31:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 32f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 32f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 34f\n"
       "32:"  // Height 3: no bias
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
       "b 34f\n"
       "33:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -656,102 +656,102 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "34:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "35:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 37f\n"
       "36:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "37:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 39f\n"
       "38:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "cmp x26, #0x4\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "cmp x27, #0x4\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -760,109 +760,109 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[3]\n"
       "bgt 38b\n"
       "39:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "ble 40f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "ble 40f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x12, x12, #1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "ble 40f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x11, x11, #1\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
+      "addvl x10, x10, #1\n"
       "addvl x9, x9, #1\n"
-      "addvl x28, x28, #1\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
       "fmla z15.s, z7.s, z1.s[3]\n"
       "fmla z19.s, z7.s, z2.s[3]\n"
       "40:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 41f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -888,71 +888,71 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z18.s, p5/M, z18.s, z0.s\n"
       "fmax z19.s, p5/M, z19.s, z0.s\n"
       "41:"  // Height 3: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "42:"  // Height 3: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 30b\n"
       "b 86f\n"
       "43:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 45f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "45:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 46f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 46f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -962,26 +962,26 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "b 48f\n"
       "46:"  // Height 4: no bias
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "b 48f\n"
       "47:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -1001,121 +1001,121 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "48:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 51f\n"
       "50:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "51:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 53f\n"
       "52:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "cmp x26, #0x4\n"
-      "add x25, x25, #0x10\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "add x24, x24, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z22.s, z6.s, z3.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "fmla z23.s, z7.s, z3.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z22.s, z6.s, z3.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "fmla z23.s, z7.s, z3.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z22.s, z6.s, z3.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "fmla z23.s, z7.s, z3.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -1126,28 +1126,28 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[3]\n"
       "bgt 52b\n"
       "53:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
@@ -1157,25 +1157,25 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[0]\n"
       "fmla z23.s, z7.s, z3.s[0]\n"
       "ble 54f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z22.s, z6.s, z3.s[1]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
@@ -1183,25 +1183,25 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[1]\n"
       "fmla z23.s, z7.s, z3.s[1]\n"
       "ble 54f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "subs x26, x26, #0x1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x10, x10, #1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
-      "addvl x28, x28, #1\n"
+      "addvl x9, x9, #1\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z22.s, z6.s, z3.s[2]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
@@ -1209,22 +1209,22 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[2]\n"
       "fmla z23.s, z7.s, z3.s[2]\n"
       "ble 54f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x11, x11, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x12, x12, #1\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
+      "addvl x11, x11, #1\n"
       "addvl x10, x10, #1\n"
-      "addvl x9, x9, #1\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -1234,19 +1234,19 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[3]\n"
       "fmla z23.s, z7.s, z3.s[3]\n"
       "54:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 55f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -1280,75 +1280,75 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z22.s, p5/M, z22.s, z0.s\n"
       "fmax z23.s, p5/M, z23.s, z0.s\n"
       "55:"  // Height 4: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
       "56:"  // Height 4: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 44b\n"
       "b 86f\n"
       "57:"  // Height 5
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "58:"  // Height 5: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 59f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 59f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 59f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "59:"  // Height 5: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 60f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 60f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1362,31 +1362,31 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "b 62f\n"
       "60:"  // Height 5: no bias
       "tbz %x[flags], #0, 61f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
       "b 62f\n"
       "61:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -1410,140 +1410,140 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "62:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "63:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 64f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 65f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 65f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 65f\n"
       "64:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "65:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 67f\n"
       "66:"  // Height 5: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "cmp x26, #0x4\n"
-      "add x25, x25, #0x10\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "add x23, x23, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "add x24, x24, #0x10\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z22.s, z6.s, z3.s[0]\n"
       "fmla z26.s, z6.s, z4.s[0]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "fmla z23.s, z7.s, z3.s[0]\n"
       "fmla z27.s, z7.s, z4.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
       "fmla z25.s, z7.s, z4.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z22.s, z6.s, z3.s[1]\n"
       "fmla z26.s, z6.s, z4.s[1]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "fmla z23.s, z7.s, z3.s[1]\n"
       "fmla z27.s, z7.s, z4.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
       "fmla z25.s, z7.s, z4.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z22.s, z6.s, z3.s[2]\n"
       "fmla z26.s, z6.s, z4.s[2]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "fmla z23.s, z7.s, z3.s[2]\n"
       "fmla z27.s, z7.s, z4.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
       "fmla z25.s, z7.s, z4.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -1556,31 +1556,31 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[3]\n"
       "bgt 66b\n"
       "67:"  // Height 5: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
@@ -1592,25 +1592,25 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[0]\n"
       "fmla z27.s, z7.s, z4.s[0]\n"
       "ble 68f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
       "fmla z25.s, z7.s, z4.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
@@ -1622,25 +1622,25 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[1]\n"
       "fmla z27.s, z7.s, z4.s[1]\n"
       "ble 68f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
       "fmla z25.s, z7.s, z4.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
@@ -1652,24 +1652,24 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[2]\n"
       "fmla z27.s, z7.s, z4.s[2]\n"
       "ble 68f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
       "fmla z25.s, z7.s, z4.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -1681,20 +1681,20 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[3]\n"
       "fmla z27.s, z7.s, z4.s[3]\n"
       "68:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 63b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 69f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -1736,82 +1736,82 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z26.s, p5/M, z26.s, z0.s\n"
       "fmax z27.s, p5/M, z27.s, z0.s\n"
       "69:"  // Height 5: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "70:"  // Height 5: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 58b\n"
       "b 86f\n"
       "71:"  // Height 6
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x18\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "72:"  // Height 6: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #2\n"
-      "cntw x20, ALL, MUL #3\n"
-      "add x9, x10, x19, LSL #2\n"
-      "add x28, x9, x19, LSL #2\n"
-      "add x19, x28, x19, LSL #2\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #2\n"
+      "cntw x21, ALL, MUL #3\n"
+      "add x10, x11, x20, LSL #2\n"
+      "add x9, x10, x20, LSL #2\n"
+      "add x20, x9, x20, LSL #2\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 73f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 73f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 73f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "73:"  // Height 6: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 74f\n"
-      "ld1w { z8.s }, p5/Z, [x14]\n"
-      "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 74f\n"
+      "ld1w { z8.s }, p5/Z, [x15]\n"
+      "ld1w { z9.s }, p5/Z, [x15, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x15, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
-      "addvl x14, x14, #4\n"
+      "addvl x15, x15, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1829,36 +1829,36 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "b 76f\n"
       "74:"  // Height 6: no bias
       "tbz %x[flags], #0, 75f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p4/Z, [x12]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x13]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "b 76f\n"
       "75:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1886,159 +1886,159 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "76:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "77:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 78f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 79f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 79f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 79f\n"
       "78:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "79:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 81f\n"
       "80:"  // Height 6: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "cmp x26, #0x4\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
       "add x25, x25, #0x10\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
       "add x24, x24, #0x10\n"
-      "add x23, x23, #0x10\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
+      "add x23, x23, #0x10\n"
       "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z28.s, z6.s, z5.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "add x20, x20, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "add x21, x21, #0x10\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
       "fmla z29.s, z7.s, z5.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z22.s, z6.s, z3.s[0]\n"
       "fmla z26.s, z6.s, z4.s[0]\n"
       "fmla z30.s, z6.s, z5.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "fmla z23.s, z7.s, z3.s[0]\n"
       "fmla z27.s, z7.s, z4.s[0]\n"
       "fmla z31.s, z7.s, z5.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
       "fmla z28.s, z6.s, z5.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
       "fmla z25.s, z7.s, z4.s[1]\n"
       "fmla z29.s, z7.s, z5.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z22.s, z6.s, z3.s[1]\n"
       "fmla z26.s, z6.s, z4.s[1]\n"
       "fmla z30.s, z6.s, z5.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "fmla z23.s, z7.s, z3.s[1]\n"
       "fmla z27.s, z7.s, z4.s[1]\n"
       "fmla z31.s, z7.s, z5.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
       "fmla z28.s, z6.s, z5.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
       "fmla z25.s, z7.s, z4.s[2]\n"
       "fmla z29.s, z7.s, z5.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z22.s, z6.s, z3.s[2]\n"
       "fmla z26.s, z6.s, z4.s[2]\n"
       "fmla z30.s, z6.s, z5.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "fmla z23.s, z7.s, z3.s[2]\n"
       "fmla z27.s, z7.s, z4.s[2]\n"
       "fmla z31.s, z7.s, z5.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "addvl x11, x11, #4\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
       "fmla z28.s, z6.s, z5.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
       "fmla z25.s, z7.s, z4.s[3]\n"
       "fmla z29.s, z7.s, z5.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -2053,34 +2053,34 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z31.s, z7.s, z5.s[3]\n"
       "bgt 80b\n"
       "81:"  // Height 6: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z28.s, z6.s, z5.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
       "fmla z29.s, z7.s, z5.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
@@ -2094,27 +2094,27 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[0]\n"
       "fmla z31.s, z7.s, z5.s[0]\n"
       "ble 82f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
       "fmla z28.s, z6.s, z5.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
       "fmla z25.s, z7.s, z4.s[1]\n"
       "fmla z29.s, z7.s, z5.s[1]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
@@ -2128,27 +2128,27 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[1]\n"
       "fmla z31.s, z7.s, z5.s[1]\n"
       "ble 82f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
-      "subs x26, x26, #0x1\n"
-      "addvl x11, x11, #1\n"
+      "subs x27, x27, #0x1\n"
+      "addvl x12, x12, #1\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
       "fmla z28.s, z6.s, z5.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x10, x10, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x11, x11, #1\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
-      "addvl x9, x9, #1\n"
+      "addvl x10, x10, #1\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
       "fmla z25.s, z7.s, z4.s[2]\n"
       "fmla z29.s, z7.s, z5.s[2]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
@@ -2162,26 +2162,26 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[2]\n"
       "fmla z31.s, z7.s, z5.s[2]\n"
       "ble 82f\n"
-      "ld1w { z6.s }, p5/Z, [x11]\n"
-      "ld1w { z7.s }, p5/Z, [x10]\n"
+      "ld1w { z6.s }, p5/Z, [x12]\n"
+      "ld1w { z7.s }, p5/Z, [x11]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
+      "addvl x12, x12, #1\n"
       "addvl x11, x11, #1\n"
-      "addvl x10, x10, #1\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
       "fmla z28.s, z6.s, z5.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "ld1w { z6.s }, p5/Z, [x10]\n"
+      "addvl x10, x10, #1\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
       "fmla z25.s, z7.s, z4.s[3]\n"
       "fmla z29.s, z7.s, z5.s[3]\n"
-      "ld1w { z7.s }, p5/Z, [x28]\n"
-      "addvl x28, x28, #1\n"
+      "ld1w { z7.s }, p5/Z, [x9]\n"
+      "addvl x9, x9, #1\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
       "fmla z18.s, z6.s, z2.s[3]\n"
@@ -2195,21 +2195,21 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[3]\n"
       "fmla z31.s, z7.s, z5.s[3]\n"
       "82:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 77b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x12, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x13, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 83f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -2259,50 +2259,50 @@ void sve_ffhybrid_fp32_mla_6x4VL (
       "fmax z30.s, p5/M, z30.s, z0.s\n"
       "fmax z31.s, p5/M, z31.s, z0.s\n"
       "83:"  // Height 6: No activation
-      "st1w { z8.s }, p4, [x12]\n"
-      "st1w { z9.s }, p3, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z28.s }, p4, [x20]\n"
-      "st1w { z29.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z30.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z31.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x13]\n"
+      "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+      "addvl x13, x13, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z28.s }, p4, [x21]\n"
+      "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
       "84:"  // Height 6: Writeback done
-      "decw x13, ALL, MUL #4\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #4\n"
+      "cmp x14, XZR\n"
       "bgt 72b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 86f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 85f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "85:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "86:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
index 8e3676a007..36fc9d75ca 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -103,70 +103,70 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #5\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #5\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x26, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x27, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x27, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x28, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 3f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 3f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "3:"  // Height 1: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 4f\n"
-      "ld1w { z8.s }, p7/Z, [x14]\n"
-      "ld1w { z9.s }, p7/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 4f\n"
+      "ld1w { z8.s }, p7/Z, [x15]\n"
+      "ld1w { z9.s }, p7/Z, [x15, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p7/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p7/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x15, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z12.s }, p7/Z, [x14, #4, MUL VL]\n"
-      "ld1w { z13.s }, p7/Z, [x14, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x15, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x15, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
-      "addvl x14, x14, #6\n"
+      "addvl x15, x15, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -174,16 +174,16 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "b 6f\n"
       "4:"  // Height 1: no bias
       "tbz %x[flags], #0, 5f\n"
-      "ld1w { z9.s }, p6/Z, [x12]\n"
-      "ld1w { z10.s }, p5/Z, [x12, #1, MUL VL]\n"
+      "ld1w { z9.s }, p6/Z, [x13]\n"
+      "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z11.s }, p4/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x12, #3, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z13.s }, p2/Z, [x12, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x12, #5, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
       "zip1 z11.d, z12.d, z17.d\n"
@@ -207,102 +207,102 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "6:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "7:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 8f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 9f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 9f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 9f\n"
       "8:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "9:"  // Height 1: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 11f\n"
       "10:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x4\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
       "addvl x28, x28, #2\n"
       "addvl x27, x27, #2\n"
-      "addvl x26, x26, #2\n"
       "bgt 10b\n"
       "11:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
       "addvl x28, x28, #2\n"
       "addvl x27, x27, #2\n"
-      "addvl x26, x26, #2\n"
       "12:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 7b\n"
       "uzp1 z8.d, z8.d, z14.d\n"
       "uzp1 z9.d, z9.d, z15.d\n"
@@ -311,10 +311,10 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z12.d, z12.d, z18.d\n"
       "uzp1 z13.d, z13.d, z19.d\n"
       "tbz %x[flags], #1, 13f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z8.s, p7/M, z8.s, z1.s\n"
       "fmin z9.s, p7/M, z9.s, z1.s\n"
       "fmin z10.s, p7/M, z10.s, z1.s\n"
@@ -328,83 +328,83 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z12.s, p7/M, z12.s, z0.s\n"
       "fmax z13.s, p7/M, z13.s, z0.s\n"
       "13:"  // Height 1: No activation
-      "st1w { z8.s }, p6, [x12]\n"
-      "st1w { z9.s }, p5, [x12, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x12, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x12, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x12, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x12, #5, MUL VL]\n"
-      "addvl x12, x12, #6\n"
+      "st1w { z8.s }, p6, [x13]\n"
+      "st1w { z9.s }, p5, [x13, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x13, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x13, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x13, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x13, #5, MUL VL]\n"
+      "addvl x13, x13, #6\n"
       "14:"  // Height 1: Writeback done
-      "decw x13, ALL, MUL #6\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #6\n"
+      "cmp x14, XZR\n"
       "bgt 2b\n"
       "b 58f\n"
       "15:"  // Height 2
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #5\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #5\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x26, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x27, x12\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x27, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x28, x12\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 17f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 17f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "17:"  // Height 2: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 18f\n"
-      "ld1w { z8.s }, p7/Z, [x14]\n"
-      "ld1w { z9.s }, p7/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 18f\n"
+      "ld1w { z8.s }, p7/Z, [x15]\n"
+      "ld1w { z9.s }, p7/Z, [x15, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p7/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p7/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x15, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z12.s }, p7/Z, [x14, #4, MUL VL]\n"
-      "ld1w { z13.s }, p7/Z, [x14, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x15, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x15, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
-      "addvl x14, x14, #6\n"
+      "addvl x15, x15, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -412,26 +412,26 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "b 20f\n"
       "18:"  // Height 2: no bias
       "tbz %x[flags], #0, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x12]\n"
-      "ld1w { z10.s }, p5/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x12, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x12, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x13]\n"
+      "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
       "zip1 z12.d, z13.d, z18.d\n"
@@ -453,119 +453,119 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "20:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "21:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 22f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 23f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 23f\n"
       "22:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
       "23:"  // Height 2: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 25f\n"
       "24:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
       "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
       "addvl x28, x28, #2\n"
       "addvl x27, x27, #2\n"
-      "addvl x26, x26, #2\n"
       "bgt 24b\n"
       "25:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
+      "addvl x12, x12, #2\n"
       "addvl x11, x11, #2\n"
-      "addvl x10, x10, #2\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
+      "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
       "addvl x28, x28, #2\n"
       "addvl x27, x27, #2\n"
-      "addvl x26, x26, #2\n"
       "26:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 21b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
-      "add x22, x12, x19, LSL #2\n"
+      "add x23, x13, x20, LSL #2\n"
       "uzp1 z14.d, z9.d, z15.d\n"
       "uzp2 z9.d, z9.d, z15.d\n"
       "uzp1 z15.d, z10.d, z16.d\n"
@@ -577,10 +577,10 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z18.d, z13.d, z19.d\n"
       "uzp2 z13.d, z13.d, z19.d\n"
       "tbz %x[flags], #1, 27f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -606,89 +606,89 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z12.s, p7/M, z12.s, z0.s\n"
       "fmax z13.s, p7/M, z13.s, z0.s\n"
       "27:"  // Height 2: No activation
-      "st1w { z4.s }, p6, [x12]\n"
-      "st1w { z14.s }, p5, [x12, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x12, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x12, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x12, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x12, #5, MUL VL]\n"
-      "addvl x12, x12, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x13]\n"
+      "st1w { z14.s }, p5, [x13, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x13, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x13, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x13, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x13, #5, MUL VL]\n"
+      "addvl x13, x13, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
       "28:"  // Height 2: Writeback done
-      "decw x13, ALL, MUL #6\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #6\n"
+      "cmp x14, XZR\n"
       "bgt 16b\n"
       "b 58f\n"
       "29:"  // Height 3
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x14, %x[bias]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x12, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x15, %x[bias]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x13, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #5\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #5\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x26, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x27, x12\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x27, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x28, x12\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 31f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 31f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "31:"  // Height 3: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 32f\n"
-      "ld1w { z8.s }, p7/Z, [x14]\n"
-      "ld1w { z9.s }, p7/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 32f\n"
+      "ld1w { z8.s }, p7/Z, [x15]\n"
+      "ld1w { z9.s }, p7/Z, [x15, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p7/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p7/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x15, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z12.s }, p7/Z, [x14, #4, MUL VL]\n"
-      "ld1w { z13.s }, p7/Z, [x14, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x15, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x15, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
-      "addvl x14, x14, #6\n"
+      "addvl x15, x15, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -708,39 +708,39 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "b 34f\n"
       "32:"  // Height 3: no bias
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x12]\n"
-      "ld1w { z10.s }, p5/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x12, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x12, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x13]\n"
+      "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
-      "ld1w { z21.s }, p6/Z, [x21]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
+      "ld1w { z21.s }, p6/Z, [x22]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
-      "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n"
       "zip1 z12.d, z13.d, z18.d\n"
       "zip2 z18.d, z13.d, z18.d\n"
-      "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n"
       "zip1 z13.d, z20.d, z19.d\n"
       "zip2 z19.d, z20.d, z19.d\n"
-      "ld1w { z4.s }, p1/Z, [x21, #5, MUL VL]\n"
+      "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n"
       "zip1 z20.d, z21.d, z26.d\n"
       "zip2 z26.d, z21.d, z26.d\n"
       "zip1 z21.d, z22.d, z27.d\n"
@@ -780,137 +780,137 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "34:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "35:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 37f\n"
       "36:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "37:"  // Height 3: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 39f\n"
       "38:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "sub x24, x24, #0x4\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "add x21, x21, #0x10\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x9, x9, #2\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
+      "addvl x9, x9, #2\n"
       "addvl x28, x28, #2\n"
-      "addvl x27, x27, #2\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "addvl x26, x26, #2\n"
+      "addvl x27, x27, #2\n"
       ".inst 0x6466e459  // bfmmla z25.s, z2.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "bgt 38b\n"
       "39:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "addvl x28, x28, #2\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "addvl x27, x27, #2\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "addvl x28, x28, #2\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x26, x26, #2\n"
+      "addvl x27, x27, #2\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -919,17 +919,17 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "40:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
       "uzp1 z14.d, z9.d, z15.d\n"
       "uzp2 z9.d, z9.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 z15.d, z10.d, z16.d\n"
       "uzp2 z10.d, z10.d, z16.d\n"
       "uzp1 z16.d, z11.d, z17.d\n"
@@ -945,10 +945,10 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z24.d, z24.d, z30.d\n"
       "uzp1 z25.d, z25.d, z31.d\n"
       "tbz %x[flags], #1, 41f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -986,98 +986,98 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z24.s, p7/M, z24.s, z0.s\n"
       "fmax z25.s, p7/M, z25.s, z0.s\n"
       "41:"  // Height 3: No activation
-      "st1w { z4.s }, p6, [x12]\n"
-      "st1w { z14.s }, p5, [x12, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x12, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x12, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x12, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x12, #5, MUL VL]\n"
-      "addvl x12, x12, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
-      "st1w { z20.s }, p6, [x21]\n"
-      "st1w { z21.s }, p5, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p4, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p3, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p2, [x21, #4, MUL VL]\n"
-      "st1w { z25.s }, p1, [x21, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x13]\n"
+      "st1w { z14.s }, p5, [x13, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x13, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x13, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x13, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x13, #5, MUL VL]\n"
+      "addvl x13, x13, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
+      "st1w { z20.s }, p6, [x22]\n"
+      "st1w { z21.s }, p5, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p4, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p3, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p2, [x22, #4, MUL VL]\n"
+      "st1w { z25.s }, p1, [x22, #5, MUL VL]\n"
       "42:"  // Height 3: Writeback done
-      "decw x13, ALL, MUL #6\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #6\n"
+      "cmp x14, XZR\n"
       "bgt 30b\n"
       "b 58f\n"
       "43:"  // Height 4
-      "ldr x19, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x20, #0x10\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x14, %x[bias]\n"
-      "mov x12, %x[output_ptr]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x14, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x21, #0x10\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x15, %x[bias]\n"
+      "mov x13, %x[output_ptr]\n"
+      "madd %x[output_ptr], x20, x21, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
-      "ldr x11, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "add x10, x11, x19, LSL #1\n"
-      "add x9, x10, x19, LSL #1\n"
-      "add x28, x9, x19, LSL #1\n"
-      "cntw x20, ALL, MUL #5\n"
-      "add x27, x28, x19, LSL #1\n"
-      "add x26, x27, x19, LSL #1\n"
-      "add x19, x26, x19, LSL #1\n"
-      "cmp x13, x20\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "add x11, x12, x20, LSL #1\n"
+      "add x10, x11, x20, LSL #1\n"
+      "add x9, x10, x20, LSL #1\n"
+      "cntw x21, ALL, MUL #5\n"
+      "add x28, x9, x20, LSL #1\n"
+      "add x27, x28, x20, LSL #1\n"
+      "add x20, x27, x20, LSL #1\n"
+      "cmp x14, x21\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x26, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x27, x12\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x27, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x28, x12\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x28, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x9, x12\n"
       "bgt 45f\n"
-      "decw x20\n"
-      "cmp x13, x20\n"
-      "mov x9, x11\n"
+      "decw x21\n"
+      "cmp x14, x21\n"
+      "mov x10, x12\n"
       "bgt 45f\n"
-      "mov x10, x11\n"
+      "mov x11, x12\n"
       "45:"  // Height 4: B setup done
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x13\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x14, 46f\n"
-      "ld1w { z8.s }, p7/Z, [x14]\n"
-      "ld1w { z9.s }, p7/Z, [x14, #1, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x14\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x14\n"
+      "cbz x15, 46f\n"
+      "ld1w { z8.s }, p7/Z, [x15]\n"
+      "ld1w { z9.s }, p7/Z, [x15, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p7/Z, [x14, #2, MUL VL]\n"
-      "ld1w { z11.s }, p7/Z, [x14, #3, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x15, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x15, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z12.s }, p7/Z, [x14, #4, MUL VL]\n"
-      "ld1w { z13.s }, p7/Z, [x14, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x15, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x15, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
-      "addvl x14, x14, #6\n"
+      "addvl x15, x15, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -1097,52 +1097,52 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "b 48f\n"
       "46:"  // Height 4: no bias
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x12]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p5/Z, [x12, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x12, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x12, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x12, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x12, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x13]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
-      "ld1w { z21.s }, p6/Z, [x21]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
+      "ld1w { z21.s }, p6/Z, [x22]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
-      "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n"
       "zip1 z12.d, z13.d, z18.d\n"
       "zip2 z18.d, z13.d, z18.d\n"
-      "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n"
       "zip1 z13.d, z20.d, z19.d\n"
       "zip2 z19.d, z20.d, z19.d\n"
-      "ld1w { z4.s }, p1/Z, [x21, #5, MUL VL]\n"
-      "ld1w { z26.s }, p6/Z, [x20]\n"
+      "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z26.s }, p6/Z, [x21]\n"
       "zip1 z20.d, z21.d, z26.d\n"
       "zip2 z26.d, z21.d, z26.d\n"
-      "ld1w { z27.s }, p5/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z27.s }, p5/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21, #2, MUL VL]\n"
       "zip1 z21.d, z22.d, z27.d\n"
       "zip2 z27.d, z22.d, z27.d\n"
-      "ld1w { z29.s }, p3/Z, [x20, #3, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #4, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #4, MUL VL]\n"
       "zip1 z22.d, z23.d, z28.d\n"
       "zip2 z28.d, z23.d, z28.d\n"
-      "ld1w { z31.s }, p1/Z, [x20, #5, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x21, #5, MUL VL]\n"
       "zip1 z23.d, z24.d, z29.d\n"
       "zip2 z29.d, z24.d, z29.d\n"
       "zip1 z24.d, z25.d, z30.d\n"
@@ -1176,149 +1176,149 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "48:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "49:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 51f\n"
       "50:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "51:"  // Height 4: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 53f\n"
       "52:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
-      "ld1rqw { z3.s }, p0/Z, [x20]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
+      "ld1rqw { z3.s }, p0/Z, [x21]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
       ".inst 0x658abc63  // bfcvt z3.h, p7/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
-      "sub x24, x24, #0x4\n"
+      "sub x25, x25, #0x4\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "add x21, x21, #0x10\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "add x20, x20, #0x10\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
-      "addvl x11, x11, #2\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
+      "addvl x12, x12, #2\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x10, x10, #2\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
+      "addvl x10, x10, #2\n"
       "addvl x9, x9, #2\n"
-      "addvl x28, x28, #2\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "addvl x28, x28, #2\n"
       "addvl x27, x27, #2\n"
-      "addvl x26, x26, #2\n"
       ".inst 0x6466e459  // bfmmla z25.s, z2.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "bgt 52b\n"
       "53:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
-      "ld1rqw { z3.s }, p0/Z, [x20]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
+      "ld1rqw { z3.s }, p0/Z, [x21]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
       ".inst 0x658abc63  // bfcvt z3.h, p7/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x11]\n"
-      "ld1h { z5.h }, p7/Z, [x11, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x12]\n"
+      "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z6.h }, p7/Z, [x10]\n"
-      "ld1h { z7.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x11]\n"
+      "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
-      "addvl x11, x11, #2\n"
+      "addvl x12, x12, #2\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x9]\n"
+      "ld1h { z4.h }, p7/Z, [x10]\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x9, #1, MUL VL]\n"
-      "addvl x10, x10, #2\n"
+      "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n"
+      "addvl x11, x11, #2\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x28]\n"
-      "addvl x9, x9, #2\n"
+      "ld1h { z6.h }, p7/Z, [x9]\n"
+      "addvl x10, x10, #2\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "addvl x28, x28, #2\n"
+      "addvl x9, x9, #2\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "addvl x27, x27, #2\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "addvl x28, x28, #2\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x26]\n"
+      "ld1h { z6.h }, p7/Z, [x27]\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x26, #1, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x26, x26, #2\n"
+      "addvl x27, x27, #2\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -1327,17 +1327,17 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "54:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x12, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x13, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
       "uzp1 z14.d, z9.d, z15.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z15.d\n"
       "uzp1 z15.d, z10.d, z16.d\n"
       "uzp2 z10.d, z10.d, z16.d\n"
@@ -1360,10 +1360,10 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z30.d, z25.d, z31.d\n"
       "uzp2 z25.d, z25.d, z31.d\n"
       "tbz %x[flags], #1, 55f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -1413,50 +1413,50 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z24.s, p7/M, z24.s, z0.s\n"
       "fmax z25.s, p7/M, z25.s, z0.s\n"
       "55:"  // Height 4: No activation
-      "st1w { z4.s }, p6, [x12]\n"
-      "st1w { z14.s }, p5, [x12, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x12, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x12, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x12, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x12, #5, MUL VL]\n"
-      "addvl x12, x12, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
-      "st1w { z19.s }, p6, [x21]\n"
-      "st1w { z26.s }, p5, [x21, #1, MUL VL]\n"
-      "st1w { z27.s }, p4, [x21, #2, MUL VL]\n"
-      "st1w { z28.s }, p3, [x21, #3, MUL VL]\n"
-      "st1w { z29.s }, p2, [x21, #4, MUL VL]\n"
-      "st1w { z30.s }, p1, [x21, #5, MUL VL]\n"
-      "st1w { z20.s }, p6, [x20]\n"
-      "st1w { z21.s }, p5, [x20, #1, MUL VL]\n"
-      "st1w { z22.s }, p4, [x20, #2, MUL VL]\n"
-      "st1w { z23.s }, p3, [x20, #3, MUL VL]\n"
-      "st1w { z24.s }, p2, [x20, #4, MUL VL]\n"
-      "st1w { z25.s }, p1, [x20, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x13]\n"
+      "st1w { z14.s }, p5, [x13, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x13, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x13, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x13, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x13, #5, MUL VL]\n"
+      "addvl x13, x13, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
+      "st1w { z19.s }, p6, [x22]\n"
+      "st1w { z26.s }, p5, [x22, #1, MUL VL]\n"
+      "st1w { z27.s }, p4, [x22, #2, MUL VL]\n"
+      "st1w { z28.s }, p3, [x22, #3, MUL VL]\n"
+      "st1w { z29.s }, p2, [x22, #4, MUL VL]\n"
+      "st1w { z30.s }, p1, [x22, #5, MUL VL]\n"
+      "st1w { z20.s }, p6, [x21]\n"
+      "st1w { z21.s }, p5, [x21, #1, MUL VL]\n"
+      "st1w { z22.s }, p4, [x21, #2, MUL VL]\n"
+      "st1w { z23.s }, p3, [x21, #3, MUL VL]\n"
+      "st1w { z24.s }, p2, [x21, #4, MUL VL]\n"
+      "st1w { z25.s }, p1, [x21, #5, MUL VL]\n"
       "56:"  // Height 4: Writeback done
-      "decw x13, ALL, MUL #6\n"
-      "cmp x13, XZR\n"
+      "decw x14, ALL, MUL #6\n"
+      "cmp x14, XZR\n"
       "bgt 44b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 58f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "57:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "58:"  // Exit
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp
index 1f1e08d3dd..7649336c36 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -53,33 +53,33 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x23, %x[Apanel]\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x24, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "cntw x22, ALL, MUL #2\n"
-      "add x21, x25, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "cmp x24, x22\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov %x[Apanel], x23\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "cntw x23, ALL, MUL #2\n"
+      "add x22, x26, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "cmp x25, x23\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov %x[Apanel], x24\n"
       "bgt 3f\n"
-      "decw x22\n"
-      "cmp x24, x22\n"
-      "mov x20, x25\n"
+      "decw x23\n"
+      "cmp x25, x23\n"
+      "mov x21, x26\n"
       "bgt 3f\n"
-      "mov x21, x25\n"
+      "mov x22, x26\n"
       "3:"  // B setup done
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
       "mov z10.b, #0x0\n"
-      "ld1h { z4.h }, p0/Z, [x25]\n"
+      "ld1h { z4.h }, p0/Z, [x26]\n"
       "mov z11.b, #0x0\n"
       "mov z12.b, #0x0\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
@@ -88,13 +88,13 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "mov z15.b, #0x0\n"
       "mov z16.b, #0x0\n"
-      "ld1h { z5.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x26, #1, MUL VL]\n"
       "mov z17.b, #0x0\n"
       "mov z18.b, #0x0\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
       "mov z19.b, #0x0\n"
       "mov z20.b, #0x0\n"
-      "addvl x25, x25, #2\n"
+      "addvl x26, x26, #2\n"
       "mov z21.b, #0x0\n"
       "mov z22.b, #0x0\n"
       "add %x[Apanel], %x[Apanel], #0x30\n"
@@ -114,25 +114,25 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       ".inst 0x6465e40b  // bfmmla z11.s, z0.h, z5.h\n"
       ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e431  // bfmmla z17.s, z1.h, z5.h\n"
-      "ld1h { z6.h }, p0/Z, [x21]\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e457  // bfmmla z23.s, z2.h, z5.h\n"
-      "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x6464e47a  // bfmmla z26.s, z3.h, z4.h\n"
       ".inst 0x6465e47d  // bfmmla z29.s, z3.h, z5.h\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x21]\n"
+      "ld1h { z5.h }, p0/Z, [x21, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6467e432  // bfmmla z18.s, z1.h, z7.h\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       ".inst 0x6466e47b  // bfmmla z27.s, z3.h, z6.h\n"
       ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
-      "ld1h { z6.h }, p0/Z, [x25]\n"
+      "ld1h { z6.h }, p0/Z, [x26]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e40d  // bfmmla z13.s, z0.h, z5.h\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #16]\n"
@@ -141,7 +141,7 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #32]\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e459  // bfmmla z25.s, z2.h, z5.h\n"
-      "ld1h { z7.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x26, #1, MUL VL]\n"
       ".inst 0x6464e47c  // bfmmla z28.s, z3.h, z4.h\n"
       ".inst 0x6465e47f  // bfmmla z31.s, z3.h, z5.h\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #48]\n"
@@ -150,25 +150,25 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6467e431  // bfmmla z17.s, z1.h, z7.h\n"
-      "ld1h { z4.h }, p0/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
-      "ld1h { z5.h }, p0/Z, [x21, #3, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x6466e47a  // bfmmla z26.s, z3.h, z6.h\n"
       ".inst 0x6467e47d  // bfmmla z29.s, z3.h, z7.h\n"
-      "ld1h { z6.h }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x21, #3, MUL VL]\n"
       ".inst 0x6464e409  // bfmmla z9.s, z0.h, z4.h\n"
       ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
       ".inst 0x6464e42f  // bfmmla z15.s, z1.h, z4.h\n"
       ".inst 0x6465e432  // bfmmla z18.s, z1.h, z5.h\n"
-      "addvl x21, x21, #4\n"
+      "addvl x22, x22, #4\n"
       ".inst 0x6464e455  // bfmmla z21.s, z2.h, z4.h\n"
       ".inst 0x6465e458  // bfmmla z24.s, z2.h, z5.h\n"
-      "addvl x20, x20, #4\n"
+      "addvl x21, x21, #4\n"
       ".inst 0x6464e47b  // bfmmla z27.s, z3.h, z4.h\n"
       ".inst 0x6465e47e  // bfmmla z30.s, z3.h, z5.h\n"
-      "ld1h { z4.h }, p0/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x26, #2, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #80]\n"
@@ -177,12 +177,12 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #96]\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6467e459  // bfmmla z25.s, z2.h, z7.h\n"
-      "ld1h { z5.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x26, #3, MUL VL]\n"
       ".inst 0x6466e47c  // bfmmla z28.s, z3.h, z6.h\n"
       ".inst 0x6467e47f  // bfmmla z31.s, z3.h, z7.h\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #112]\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "addvl x25, x25, #4\n"
+      "addvl x26, x26, #4\n"
       "bge 4b\n"
       "5:"  // main loop skip
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n"
@@ -190,14 +190,14 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       ".inst 0x6465e40b  // bfmmla z11.s, z0.h, z5.h\n"
       ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e431  // bfmmla z17.s, z1.h, z5.h\n"
-      "ld1h { z6.h }, p0/Z, [x21]\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e457  // bfmmla z23.s, z2.h, z5.h\n"
-      "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x6464e47a  // bfmmla z26.s, z3.h, z4.h\n"
       ".inst 0x6465e47d  // bfmmla z29.s, z3.h, z5.h\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x21]\n"
+      "ld1h { z5.h }, p0/Z, [x21, #1, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
@@ -205,10 +205,10 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
-      "addvl x21, x21, #2\n"
+      "addvl x22, x22, #2\n"
       ".inst 0x6466e47b  // bfmmla z27.s, z3.h, z6.h\n"
       ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
-      "addvl x20, x20, #2\n"
+      "addvl x21, x21, #2\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e40d  // bfmmla z13.s, z0.h, z5.h\n"
       ".inst 0x6464e430  // bfmmla z16.s, z1.h, z4.h\n"
@@ -217,25 +217,25 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       ".inst 0x6465e459  // bfmmla z25.s, z2.h, z5.h\n"
       ".inst 0x6464e47c  // bfmmla z28.s, z3.h, z4.h\n"
       ".inst 0x6465e47f  // bfmmla z31.s, z3.h, z5.h\n"
-      "cbz x19, 6f\n"
-      "ld1h { z6.h }, p0/Z, [x25]\n"
+      "cbz x20, 6f\n"
+      "ld1h { z6.h }, p0/Z, [x26]\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
-      "ld1h { z7.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x26, #1, MUL VL]\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6467e431  // bfmmla z17.s, z1.h, z7.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z4.h }, p0/Z, [x21]\n"
+      "ld1h { z4.h }, p0/Z, [x22]\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       ".inst 0x6466e47a  // bfmmla z26.s, z3.h, z6.h\n"
-      "ld1h { z5.h }, p0/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x6467e47d  // bfmmla z29.s, z3.h, z7.h\n"
-      "ld1h { z6.h }, p0/Z, [x20]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x21]\n"
+      "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n"
       ".inst 0x6464e409  // bfmmla z9.s, z0.h, z4.h\n"
       ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
@@ -254,7 +254,7 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       ".inst 0x6466e47c  // bfmmla z28.s, z3.h, z6.h\n"
       ".inst 0x6467e47f  // bfmmla z31.s, z3.h, z7.h\n"
       "6:"  // multiply loop done
-      "decw x24, ALL, MUL #3\n"
+      "decw x25, ALL, MUL #3\n"
       "uzp1 z4.d, z8.d, z11.d\n"
       "uzp2 z8.d, z8.d, z11.d\n"
       "uzp1 z11.d, z9.d, z12.d\n"
@@ -268,7 +268,7 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "uzp2 z14.d, z14.d, z17.d\n"
       "st1w { z8.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
       "uzp1 z17.d, z15.d, z18.d\n"
-      "cmp x24, XZR\n"
+      "cmp x25, XZR\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
       "uzp2 z15.d, z15.d, z18.d\n"
       "uzp1 z18.d, z16.d, z19.d\n"
@@ -311,7 +311,7 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp
index cd4da2c124..1d502f5354 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -52,39 +52,39 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x23, %x[Apanel]\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x24, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "cnth x22, ALL, MUL #2\n"
-      "add x21, x25, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "cmp x24, x22\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov %x[Apanel], x23\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "cnth x23, ALL, MUL #2\n"
+      "add x22, x26, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "cmp x25, x23\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov %x[Apanel], x24\n"
       "bgt 3f\n"
-      "dech x22\n"
-      "cmp x24, x22\n"
-      "mov x20, x25\n"
+      "dech x23\n"
+      "cmp x25, x23\n"
+      "mov x21, x26\n"
       "bgt 3f\n"
-      "mov x21, x25\n"
+      "mov x22, x26\n"
       "3:"  // B setup done
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
       "mov z10.b, #0x0\n"
-      "ld1h { z0.h }, p0/Z, [x25]\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
       "mov z11.b, #0x0\n"
       "mov z12.b, #0x0\n"
-      "ld1h { z1.h }, p0/Z, [x21]\n"
+      "ld1h { z1.h }, p0/Z, [x22]\n"
       "mov z13.b, #0x0\n"
       "mov z14.b, #0x0\n"
-      "ld1h { z2.h }, p0/Z, [x20]\n"
+      "ld1h { z2.h }, p0/Z, [x21]\n"
       "mov z15.b, #0x0\n"
       "mov z16.b, #0x0\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n"
@@ -110,7 +110,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "4:"  // main loop head
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z10.h, p0/M, z2.h, z3.h\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z11.h, p0/M, z0.h, z4.h\n"
@@ -119,7 +119,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
@@ -139,11 +139,11 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "fmla z28.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z29.h, p0/M, z0.h, z6.h\n"
-      "ld1h { z0.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z0.h }, p0/Z, [x26, #1, MUL VL]\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "ld1h { z1.h }, p0/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z2.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z2.h }, p0/Z, [x21, #1, MUL VL]\n"
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #22]\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
@@ -155,15 +155,15 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #26]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
-      "addvl x25, x25, #2\n"
+      "addvl x26, x26, #2\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #28]\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
       "fmla z18.h, p0/M, z1.h, z6.h\n"
       "fmla z19.h, p0/M, z2.h, z6.h\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #30]\n"
+      "addvl x22, x22, #2\n"
       "addvl x21, x21, #2\n"
-      "addvl x20, x20, #2\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z20.h, p0/M, z0.h, z3.h\n"
       "fmla z21.h, p0/M, z1.h, z3.h\n"
@@ -178,17 +178,17 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "fmla z28.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n"
       "fmla z29.h, p0/M, z0.h, z6.h\n"
-      "ld1h { z0.h }, p0/Z, [x25]\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "ld1h { z1.h }, p0/Z, [x21]\n"
-      "ld1h { z2.h }, p0/Z, [x20]\n"
+      "ld1h { z1.h }, p0/Z, [x22]\n"
+      "ld1h { z2.h }, p0/Z, [x21]\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n"
       "bge 4b\n"
       "5:"  // main loop skip
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
-      "addvl x25, x25, #1\n"
+      "addvl x26, x26, #1\n"
       "fmla z10.h, p0/M, z2.h, z3.h\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z11.h, p0/M, z0.h, z4.h\n"
@@ -197,7 +197,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
-      "addvl x21, x21, #1\n"
+      "addvl x22, x22, #1\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
@@ -206,7 +206,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n"
       "fmla z20.h, p0/M, z0.h, z3.h\n"
       "fmla z21.h, p0/M, z1.h, z3.h\n"
-      "addvl x20, x20, #1\n"
+      "addvl x21, x21, #1\n"
       "fmla z22.h, p0/M, z2.h, z3.h\n"
       "fmla z23.h, p0/M, z0.h, z4.h\n"
       "add %x[Apanel], %x[Apanel], #0x10\n"
@@ -218,10 +218,10 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "fmla z29.h, p0/M, z0.h, z6.h\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "cbz x19, 6f\n"
-      "ld1h { z0.h }, p0/Z, [x25]\n"
-      "ld1h { z1.h }, p0/Z, [x21]\n"
-      "ld1h { z2.h }, p0/Z, [x20]\n"
+      "cbz x20, 6f\n"
+      "ld1h { z0.h }, p0/Z, [x26]\n"
+      "ld1h { z1.h }, p0/Z, [x22]\n"
+      "ld1h { z2.h }, p0/Z, [x21]\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n"
@@ -256,9 +256,9 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
       "6:"  // multiply loop done
-      "dech x24, ALL, MUL #3\n"
+      "dech x25, ALL, MUL #3\n"
       "st1h { z8.h }, p0, [%x[Cpanel]]\n"
-      "cmp x24, XZR\n"
+      "cmp x25, XZR\n"
       "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -289,7 +289,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp
index 431c2a88f5..de219aa2bf 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -52,42 +52,42 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x23, %x[Apanel]\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x24, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "cnth x22, ALL, MUL #2\n"
-      "add x21, x25, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
-      "add x19, x20, x19, LSL #1\n"
-      "cmp x24, x22\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov %x[Apanel], x23\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "cnth x23, ALL, MUL #2\n"
+      "add x22, x26, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
+      "add x20, x21, x20, LSL #1\n"
+      "cmp x25, x23\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov %x[Apanel], x24\n"
       "bgt 3f\n"
-      "dech x22\n"
-      "cmp x24, x22\n"
-      "mov x20, x25\n"
+      "dech x23\n"
+      "cmp x25, x23\n"
+      "mov x21, x26\n"
       "bgt 3f\n"
-      "mov x21, x25\n"
+      "mov x22, x26\n"
       "3:"  // B setup done
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
       "mov z10.b, #0x0\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "mov z11.b, #0x0\n"
       "mov z12.b, #0x0\n"
-      "ld1h { z2.h }, p0/Z, [x25]\n"
+      "ld1h { z2.h }, p0/Z, [x26]\n"
       "mov z13.b, #0x0\n"
       "mov z14.b, #0x0\n"
-      "ld1h { z3.h }, p0/Z, [x21]\n"
+      "ld1h { z3.h }, p0/Z, [x22]\n"
       "mov z15.b, #0x0\n"
       "mov z16.b, #0x0\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
+      "ld1h { z4.h }, p0/Z, [x21]\n"
       "mov z17.b, #0x0\n"
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
@@ -110,43 +110,43 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z14.h, z2.h, z0.h[2]\n"
       "fmla z17.h, z2.h, z0.h[3]\n"
-      "ld1h { z5.h }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x26, #1, MUL VL]\n"
       "fmla z20.h, z2.h, z0.h[4]\n"
       "fmla z23.h, z2.h, z0.h[5]\n"
-      "ld1h { z6.h }, p0/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #1, MUL VL]\n"
       "fmla z26.h, z2.h, z0.h[6]\n"
       "fmla z29.h, z2.h, z0.h[7]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n"
       "fmla z9.h, z3.h, z0.h[0]\n"
       "fmla z12.h, z3.h, z0.h[1]\n"
-      "addvl x25, x25, #2\n"
+      "addvl x26, x26, #2\n"
       "fmla z15.h, z3.h, z0.h[2]\n"
       "fmla z18.h, z3.h, z0.h[3]\n"
-      "addvl x21, x21, #2\n"
+      "addvl x22, x22, #2\n"
       "fmla z21.h, z3.h, z0.h[4]\n"
       "fmla z24.h, z3.h, z0.h[5]\n"
-      "addvl x20, x20, #2\n"
+      "addvl x21, x21, #2\n"
       "fmla z27.h, z3.h, z0.h[6]\n"
       "fmla z30.h, z3.h, z0.h[7]\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z10.h, z4.h, z0.h[0]\n"
       "fmla z13.h, z4.h, z0.h[1]\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z16.h, z4.h, z0.h[2]\n"
       "fmla z19.h, z4.h, z0.h[3]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z22.h, z4.h, z0.h[4]\n"
       "fmla z25.h, z4.h, z0.h[5]\n"
-      "ld1h { z2.h }, p0/Z, [x25]\n"
+      "ld1h { z2.h }, p0/Z, [x26]\n"
       "fmla z28.h, z4.h, z0.h[6]\n"
       "fmla z31.h, z4.h, z0.h[7]\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.h, z5.h, z1.h[0]\n"
       "fmla z11.h, z5.h, z1.h[1]\n"
-      "ld1h { z3.h }, p0/Z, [x21]\n"
+      "ld1h { z3.h }, p0/Z, [x22]\n"
       "fmla z14.h, z5.h, z1.h[2]\n"
       "fmla z17.h, z5.h, z1.h[3]\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
+      "ld1h { z4.h }, p0/Z, [x21]\n"
       "fmla z20.h, z5.h, z1.h[4]\n"
       "fmla z23.h, z5.h, z1.h[5]\n"
       "fmla z26.h, z5.h, z1.h[6]\n"
@@ -174,13 +174,13 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
       "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla z14.h, z2.h, z0.h[2]\n"
       "fmla z17.h, z2.h, z0.h[3]\n"
-      "addvl x25, x25, #1\n"
+      "addvl x26, x26, #1\n"
       "fmla z20.h, z2.h, z0.h[4]\n"
       "fmla z23.h, z2.h, z0.h[5]\n"
-      "addvl x21, x21, #1\n"
+      "addvl x22, x22, #1\n"
       "fmla z26.h, z2.h, z0.h[6]\n"
       "fmla z29.h, z2.h, z0.h[7]\n"
-      "addvl x20, x20, #1\n"
+      "addvl x21, x21, #1\n"
       "fmla z9.h, z3.h, z0.h[0]\n"
       "fmla z12.h, z3.h, z0.h[1]\n"
       "fmla z15.h, z3.h, z0.h[2]\n"
@@ -197,12 +197,12 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
       "fmla z25.h, z4.h, z0.h[5]\n"
       "fmla z28.h, z4.h, z0.h[6]\n"
       "fmla z31.h, z4.h, z0.h[7]\n"
-      "cbz x19, 6f\n"
+      "cbz x20, 6f\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
-      "ld1h { z5.h }, p0/Z, [x25]\n"
+      "ld1h { z5.h }, p0/Z, [x26]\n"
       "fmla z8.h, z5.h, z0.h[0]\n"
-      "ld1h { z6.h }, p0/Z, [x21]\n"
-      "ld1h { z7.h }, p0/Z, [x20]\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
+      "ld1h { z7.h }, p0/Z, [x21]\n"
       "fmla z11.h, z5.h, z0.h[1]\n"
       "fmla z14.h, z5.h, z0.h[2]\n"
       "fmla z17.h, z5.h, z0.h[3]\n"
@@ -228,9 +228,9 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
       "fmla z28.h, z7.h, z0.h[6]\n"
       "fmla z31.h, z7.h, z0.h[7]\n"
       "6:"  // multiply loop done
-      "dech x24, ALL, MUL #3\n"
+      "dech x25, ALL, MUL #3\n"
       "st1h { z8.h }, p0, [%x[Cpanel]]\n"
-      "cmp x24, XZR\n"
+      "cmp x25, XZR\n"
       "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -261,7 +261,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp
index aecf7f94c9..8c8b6b0675 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -52,39 +52,39 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x23, %x[Apanel]\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x24, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "cntw x22, ALL, MUL #2\n"
-      "add x21, x25, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
-      "cmp x24, x22\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov %x[Apanel], x23\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "cntw x23, ALL, MUL #2\n"
+      "add x22, x26, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x25, x23\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov %x[Apanel], x24\n"
       "bgt 3f\n"
-      "decw x22\n"
-      "cmp x24, x22\n"
-      "mov x20, x25\n"
+      "decw x23\n"
+      "cmp x25, x23\n"
+      "mov x21, x26\n"
       "bgt 3f\n"
-      "mov x21, x25\n"
+      "mov x22, x26\n"
       "3:"  // B setup done
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
       "mov z10.b, #0x0\n"
-      "ld1w { z0.s }, p0/Z, [x25]\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
       "mov z11.b, #0x0\n"
       "mov z12.b, #0x0\n"
-      "ld1w { z1.s }, p0/Z, [x21]\n"
+      "ld1w { z1.s }, p0/Z, [x22]\n"
       "mov z13.b, #0x0\n"
       "mov z14.b, #0x0\n"
-      "ld1w { z2.s }, p0/Z, [x20]\n"
+      "ld1w { z2.s }, p0/Z, [x21]\n"
       "mov z15.b, #0x0\n"
       "mov z16.b, #0x0\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
@@ -110,7 +110,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "4:"  // main loop head
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z10.s, p0/M, z2.s, z3.s\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z11.s, p0/M, z0.s, z4.s\n"
@@ -119,7 +119,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
@@ -139,11 +139,11 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "fmla z28.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n"
       "fmla z29.s, p0/M, z0.s, z6.s\n"
-      "ld1w { z0.s }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z0.s }, p0/Z, [x26, #1, MUL VL]\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "ld1w { z1.s }, p0/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z2.s }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z2.s }, p0/Z, [x21, #1, MUL VL]\n"
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
@@ -155,15 +155,15 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
-      "addvl x25, x25, #2\n"
+      "addvl x26, x26, #2\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
       "fmla z18.s, p0/M, z1.s, z6.s\n"
       "fmla z19.s, p0/M, z2.s, z6.s\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n"
+      "addvl x22, x22, #2\n"
       "addvl x21, x21, #2\n"
-      "addvl x20, x20, #2\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       "fmla z20.s, p0/M, z0.s, z3.s\n"
       "fmla z21.s, p0/M, z1.s, z3.s\n"
@@ -178,17 +178,17 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "fmla z28.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z29.s, p0/M, z0.s, z6.s\n"
-      "ld1w { z0.s }, p0/Z, [x25]\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "ld1w { z1.s }, p0/Z, [x21]\n"
-      "ld1w { z2.s }, p0/Z, [x20]\n"
+      "ld1w { z1.s }, p0/Z, [x22]\n"
+      "ld1w { z2.s }, p0/Z, [x21]\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "bge 4b\n"
       "5:"  // main loop skip
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
-      "addvl x25, x25, #1\n"
+      "addvl x26, x26, #1\n"
       "fmla z10.s, p0/M, z2.s, z3.s\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z11.s, p0/M, z0.s, z4.s\n"
@@ -197,7 +197,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
-      "addvl x21, x21, #1\n"
+      "addvl x22, x22, #1\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
@@ -206,7 +206,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
       "fmla z20.s, p0/M, z0.s, z3.s\n"
       "fmla z21.s, p0/M, z1.s, z3.s\n"
-      "addvl x20, x20, #1\n"
+      "addvl x21, x21, #1\n"
       "fmla z22.s, p0/M, z2.s, z3.s\n"
       "fmla z23.s, p0/M, z0.s, z4.s\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
@@ -218,10 +218,10 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "fmla z29.s, p0/M, z0.s, z6.s\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "cbz x19, 6f\n"
-      "ld1w { z0.s }, p0/Z, [x25]\n"
-      "ld1w { z1.s }, p0/Z, [x21]\n"
-      "ld1w { z2.s }, p0/Z, [x20]\n"
+      "cbz x20, 6f\n"
+      "ld1w { z0.s }, p0/Z, [x26]\n"
+      "ld1w { z1.s }, p0/Z, [x22]\n"
+      "ld1w { z2.s }, p0/Z, [x21]\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
@@ -256,9 +256,9 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
       "6:"  // multiply loop done
-      "decw x24, ALL, MUL #3\n"
+      "decw x25, ALL, MUL #3\n"
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "cmp x24, XZR\n"
+      "cmp x25, XZR\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -289,7 +289,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp
index 1e9a3f119e..4a0b31daff 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Arm Limited.
+ * Copyright (c) 2022-2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -52,29 +52,29 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n"
-      "ldr x24, [%x[args_ptr], %[offsetof_N]]\n"
-      "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov x23, %x[Apanel]\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n"
+      "ldr x25, [%x[args_ptr], %[offsetof_N]]\n"
+      "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov x24, %x[Apanel]\n"
       "2:"  // Width loop
-      "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_B_stride]]\n"
-      "cntw x22, ALL, MUL #2\n"
-      "add x21, x25, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "add x19, x20, x19, LSL #2\n"
-      "cmp x24, x22\n"
-      "str x19, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
-      "mov %x[Apanel], x23\n"
+      "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n"
+      "cntw x23, ALL, MUL #2\n"
+      "add x22, x26, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
+      "cmp x25, x23\n"
+      "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n"
+      "mov %x[Apanel], x24\n"
       "bgt 3f\n"
-      "decw x22\n"
-      "cmp x24, x22\n"
-      "mov x20, x25\n"
+      "decw x23\n"
+      "cmp x25, x23\n"
+      "mov x21, x26\n"
       "bgt 3f\n"
-      "mov x21, x25\n"
+      "mov x22, x26\n"
       "3:"  // B setup done
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
-      "cmp x19, #0x2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
       "mov z10.b, #0x0\n"
@@ -84,13 +84,13 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "mov z13.b, #0x0\n"
       "mov z14.b, #0x0\n"
-      "ld1w { z4.s }, p0/Z, [x25]\n"
+      "ld1w { z4.s }, p0/Z, [x26]\n"
       "mov z15.b, #0x0\n"
       "mov z16.b, #0x0\n"
-      "ld1w { z5.s }, p0/Z, [x21]\n"
+      "ld1w { z5.s }, p0/Z, [x22]\n"
       "mov z17.b, #0x0\n"
       "mov z18.b, #0x0\n"
-      "ld1w { z6.s }, p0/Z, [x20]\n"
+      "ld1w { z6.s }, p0/Z, [x21]\n"
       "mov z19.b, #0x0\n"
       "mov z20.b, #0x0\n"
       "mov z21.b, #0x0\n"
@@ -114,25 +114,25 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #48]\n"
       "fmla z20.s, z4.s, z1.s[0]\n"
       "fmla z23.s, z4.s, z1.s[1]\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z26.s, z4.s, z1.s[2]\n"
       "fmla z29.s, z4.s, z1.s[3]\n"
-      "ld1w { z4.s }, p0/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z4.s }, p0/Z, [x26, #1, MUL VL]\n"
       "fmla z9.s, z5.s, z0.s[0]\n"
       "fmla z12.s, z5.s, z0.s[1]\n"
-      "addvl x25, x25, #2\n"
+      "addvl x26, x26, #2\n"
       "fmla z15.s, z5.s, z0.s[2]\n"
       "fmla z18.s, z5.s, z0.s[3]\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z21.s, z5.s, z1.s[0]\n"
       "fmla z24.s, z5.s, z1.s[1]\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       "fmla z27.s, z5.s, z1.s[2]\n"
       "fmla z30.s, z5.s, z1.s[3]\n"
-      "ld1w { z5.s }, p0/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z5.s }, p0/Z, [x22, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z13.s, z6.s, z0.s[1]\n"
-      "addvl x21, x21, #2\n"
+      "addvl x22, x22, #2\n"
       "fmla z16.s, z6.s, z0.s[2]\n"
       "fmla z19.s, z6.s, z0.s[3]\n"
       "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
@@ -140,8 +140,8 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z1.s[1]\n"
       "fmla z28.s, z6.s, z1.s[2]\n"
       "fmla z31.s, z6.s, z1.s[3]\n"
-      "ld1w { z6.s }, p0/Z, [x20, #1, MUL VL]\n"
-      "addvl x20, x20, #2\n"
+      "ld1w { z6.s }, p0/Z, [x21, #1, MUL VL]\n"
+      "addvl x21, x21, #2\n"
       "fmla z8.s, z4.s, z2.s[0]\n"
       "fmla z11.s, z4.s, z2.s[1]\n"
       "fmla z14.s, z4.s, z2.s[2]\n"
@@ -151,7 +151,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z23.s, z4.s, z3.s[1]\n"
       "fmla z26.s, z4.s, z3.s[2]\n"
       "fmla z29.s, z4.s, z3.s[3]\n"
-      "ld1w { z4.s }, p0/Z, [x25]\n"
+      "ld1w { z4.s }, p0/Z, [x26]\n"
       "fmla z9.s, z5.s, z2.s[0]\n"
       "fmla z12.s, z5.s, z2.s[1]\n"
       "fmla z15.s, z5.s, z2.s[2]\n"
@@ -160,7 +160,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z24.s, z5.s, z3.s[1]\n"
       "fmla z27.s, z5.s, z3.s[2]\n"
       "fmla z30.s, z5.s, z3.s[3]\n"
-      "ld1w { z5.s }, p0/Z, [x21]\n"
+      "ld1w { z5.s }, p0/Z, [x22]\n"
       "fmla z10.s, z6.s, z2.s[0]\n"
       "fmla z13.s, z6.s, z2.s[1]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
@@ -169,7 +169,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z3.s[1]\n"
       "fmla z28.s, z6.s, z3.s[2]\n"
       "fmla z31.s, z6.s, z3.s[3]\n"
-      "ld1w { z6.s }, p0/Z, [x20]\n"
+      "ld1w { z6.s }, p0/Z, [x21]\n"
       "bge 4b\n"
       "5:"  // main loop skip
       "fmla z8.s, z4.s, z0.s[0]\n"
@@ -177,13 +177,13 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z14.s, z4.s, z0.s[2]\n"
       "fmla z17.s, z4.s, z0.s[3]\n"
-      "addvl x25, x25, #1\n"
+      "addvl x26, x26, #1\n"
       "fmla z20.s, z4.s, z1.s[0]\n"
       "fmla z23.s, z4.s, z1.s[1]\n"
-      "addvl x21, x21, #1\n"
+      "addvl x22, x22, #1\n"
       "fmla z26.s, z4.s, z1.s[2]\n"
       "fmla z29.s, z4.s, z1.s[3]\n"
-      "addvl x20, x20, #1\n"
+      "addvl x21, x21, #1\n"
       "fmla z9.s, z5.s, z0.s[0]\n"
       "fmla z12.s, z5.s, z0.s[1]\n"
       "fmla z15.s, z5.s, z0.s[2]\n"
@@ -200,14 +200,14 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z1.s[1]\n"
       "fmla z28.s, z6.s, z1.s[2]\n"
       "fmla z31.s, z6.s, z1.s[3]\n"
-      "cbz x19, 6f\n"
+      "cbz x20, 6f\n"
       "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
       "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ld1w { z7.s }, p0/Z, [x25]\n"
-      "ld1w { z4.s }, p0/Z, [x21]\n"
+      "ld1w { z7.s }, p0/Z, [x26]\n"
+      "ld1w { z4.s }, p0/Z, [x22]\n"
       "fmla z8.s, z7.s, z0.s[0]\n"
-      "ld1w { z5.s }, p0/Z, [x20]\n"
+      "ld1w { z5.s }, p0/Z, [x21]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z14.s, z7.s, z0.s[2]\n"
       "fmla z17.s, z7.s, z0.s[3]\n"
@@ -232,9 +232,9 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "fmla z28.s, z5.s, z1.s[2]\n"
       "fmla z31.s, z5.s, z1.s[3]\n"
       "6:"  // multiply loop done
-      "decw x24, ALL, MUL #3\n"
+      "decw x25, ALL, MUL #3\n"
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "cmp x24, XZR\n"
+      "cmp x25, XZR\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -265,7 +265,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp
index b794c21807..f0b00e6251 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -103,32 +103,32 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 3f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -136,175 +136,175 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      ".inst 0x646840ca  // bfdot z10.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
-      ".inst 0x646840ca  // bfdot z10.s, z6.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x647040ca  // bfdot z10.s, z6.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x647040eb  // bfdot z11.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x647840ca  // bfdot z10.s, z6.h, z0.h[3]\n"
       ".inst 0x647840eb  // bfdot z11.s, z7.h, z0.h[3]\n"
+      "add x26, x26, #0x10\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646840ca  // bfdot z10.s, z6.h, z0.h[1]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647040ca  // bfdot z10.s, z6.h, z0.h[2]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x647040eb  // bfdot z11.s, z7.h, z0.h[2]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x647840ca  // bfdot z10.s, z6.h, z0.h[3]\n"
       ".inst 0x647840eb  // bfdot z11.s, z7.h, z0.h[3]\n"
+      "addvl x10, x10, #4\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "13:"  // Height 1: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 16f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 16f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "addvl x12, x12, #4\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "b 18f\n"
       "17:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -316,52 +316,52 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "19:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "21:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
       ".inst 0x646140ce  // bfdot z14.s, z6.h, z1.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
       ".inst 0x646140ef  // bfdot z15.s, z7.h, z1.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -396,156 +396,156 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647940ef  // bfdot z15.s, z7.h, z1.h[3]\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x2\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
       ".inst 0x646140ce  // bfdot z14.s, z6.h, z1.h[0]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
       ".inst 0x646140ef  // bfdot z15.s, z7.h, z1.h[0]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646840ca  // bfdot z10.s, z6.h, z0.h[1]\n"
       ".inst 0x646940ce  // bfdot z14.s, z6.h, z1.h[1]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
       ".inst 0x646940ef  // bfdot z15.s, z7.h, z1.h[1]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647040ca  // bfdot z10.s, z6.h, z0.h[2]\n"
       ".inst 0x647140ce  // bfdot z14.s, z6.h, z1.h[2]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x647040eb  // bfdot z11.s, z7.h, z0.h[2]\n"
       ".inst 0x647140ef  // bfdot z15.s, z7.h, z1.h[2]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
       ".inst 0x647940ed  // bfdot z13.s, z7.h, z1.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       ".inst 0x647840ca  // bfdot z10.s, z6.h, z0.h[3]\n"
       ".inst 0x647940ce  // bfdot z14.s, z6.h, z1.h[3]\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x647840eb  // bfdot z11.s, z7.h, z0.h[3]\n"
       ".inst 0x647940ef  // bfdot z15.s, z7.h, z1.h[3]\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 29f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 29f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "addvl x9, x9, #4\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
       "b 31f\n"
       "30:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -561,63 +561,63 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "34:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
       ".inst 0x646140ce  // bfdot z14.s, z6.h, z1.h[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x646240d2  // bfdot z18.s, z6.h, z2.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x646140ef  // bfdot z15.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f3  // bfdot z19.s, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f1  // bfdot z17.s, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -625,32 +625,32 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646840ca  // bfdot z10.s, z6.h, z0.h[1]\n"
       ".inst 0x646940ce  // bfdot z14.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d2  // bfdot z18.s, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x646940ef  // bfdot z15.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f3  // bfdot z19.s, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f1  // bfdot z17.s, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x647040ca  // bfdot z10.s, z6.h, z0.h[2]\n"
       ".inst 0x647140ce  // bfdot z14.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d2  // bfdot z18.s, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x647040eb  // bfdot z11.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x647140ef  // bfdot z15.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f3  // bfdot z19.s, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x647940ed  // bfdot z13.s, z7.h, z1.h[3]\n"
       ".inst 0x647a40f1  // bfdot z17.s, z7.h, z2.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
@@ -662,19 +662,19 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647a40f3  // bfdot z19.s, z7.h, z2.h[3]\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x2\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "addvl x10, x10, #4\n"
@@ -686,13 +686,13 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646240f3  // bfdot z19.s, z7.h, z2.h[0]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f1  // bfdot z17.s, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -705,13 +705,13 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646a40f3  // bfdot z19.s, z7.h, z2.h[1]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f1  // bfdot z17.s, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -724,12 +724,12 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647240f3  // bfdot z19.s, z7.h, z2.h[2]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647940ed  // bfdot z13.s, z7.h, z1.h[3]\n"
       ".inst 0x647a40f1  // bfdot z17.s, z7.h, z2.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -741,116 +741,116 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647940ef  // bfdot z15.s, z7.h, z1.h[3]\n"
       ".inst 0x647a40f3  // bfdot z19.s, z7.h, z2.h[3]\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 42f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 42f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "b 44f\n"
       "43:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -870,55 +870,55 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "47:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
+      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
@@ -992,21 +992,21 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647b40f7  // bfdot z23.s, z7.h, z3.h[3]\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x2\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
       ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1021,13 +1021,13 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646340f7  // bfdot z23.s, z7.h, z3.h[0]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
       ".inst 0x646b40d4  // bfdot z20.s, z6.h, z3.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f1  // bfdot z17.s, z7.h, z2.h[1]\n"
@@ -1044,13 +1044,13 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646b40f7  // bfdot z23.s, z7.h, z3.h[1]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
       ".inst 0x647340d4  // bfdot z20.s, z6.h, z3.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f1  // bfdot z17.s, z7.h, z2.h[2]\n"
@@ -1067,8 +1067,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647340f7  // bfdot z23.s, z7.h, z3.h[2]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
       ".inst 0x647b40d4  // bfdot z20.s, z6.h, z3.h[3]\n"
@@ -1088,103 +1088,103 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647a40f3  // bfdot z19.s, z7.h, z2.h[3]\n"
       ".inst 0x647b40f7  // bfdot z23.s, z7.h, z3.h[3]\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 55f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 55f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -1195,31 +1195,31 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
       "b 57f\n"
       "56:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -1243,61 +1243,61 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "60:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
-      "add x22, x22, #0x10\n"
-      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
-      "add x21, x21, #0x10\n"
       ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x646440d8  // bfdot z24.s, z6.h, z4.h[0]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       ".inst 0x646440f9  // bfdot z25.s, z7.h, z4.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1306,8 +1306,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646240d2  // bfdot z18.s, z6.h, z2.h[0]\n"
       ".inst 0x646340d6  // bfdot z22.s, z6.h, z3.h[0]\n"
       ".inst 0x646440da  // bfdot z26.s, z6.h, z4.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x646040eb  // bfdot z11.s, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x646140ef  // bfdot z15.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f3  // bfdot z19.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f7  // bfdot z23.s, z7.h, z3.h[0]\n"
@@ -1318,8 +1318,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
       ".inst 0x646b40d4  // bfdot z20.s, z6.h, z3.h[1]\n"
       ".inst 0x646c40d8  // bfdot z24.s, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f1  // bfdot z17.s, z7.h, z2.h[1]\n"
       ".inst 0x646b40f5  // bfdot z21.s, z7.h, z3.h[1]\n"
@@ -1331,8 +1331,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646a40d2  // bfdot z18.s, z6.h, z2.h[1]\n"
       ".inst 0x646b40d6  // bfdot z22.s, z6.h, z3.h[1]\n"
       ".inst 0x646c40da  // bfdot z26.s, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x646940ef  // bfdot z15.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f3  // bfdot z19.s, z7.h, z2.h[1]\n"
       ".inst 0x646b40f7  // bfdot z23.s, z7.h, z3.h[1]\n"
@@ -1343,8 +1343,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
       ".inst 0x647340d4  // bfdot z20.s, z6.h, z3.h[2]\n"
       ".inst 0x647440d8  // bfdot z24.s, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f1  // bfdot z17.s, z7.h, z2.h[2]\n"
       ".inst 0x647340f5  // bfdot z21.s, z7.h, z3.h[2]\n"
@@ -1355,8 +1355,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647240d2  // bfdot z18.s, z6.h, z2.h[2]\n"
       ".inst 0x647340d6  // bfdot z22.s, z6.h, z3.h[2]\n"
       ".inst 0x647440da  // bfdot z26.s, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x647040eb  // bfdot z11.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x647140ef  // bfdot z15.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f3  // bfdot z19.s, z7.h, z2.h[2]\n"
       ".inst 0x647340f7  // bfdot z23.s, z7.h, z3.h[2]\n"
@@ -1367,8 +1367,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
       ".inst 0x647b40d4  // bfdot z20.s, z6.h, z3.h[3]\n"
       ".inst 0x647c40d8  // bfdot z24.s, z6.h, z4.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x647940ed  // bfdot z13.s, z7.h, z1.h[3]\n"
       ".inst 0x647a40f1  // bfdot z17.s, z7.h, z2.h[3]\n"
       ".inst 0x647b40f5  // bfdot z21.s, z7.h, z3.h[3]\n"
@@ -1386,23 +1386,23 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647c40fb  // bfdot z27.s, z7.h, z4.h[3]\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x2\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
       ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
       ".inst 0x646440d8  // bfdot z24.s, z6.h, z4.h[0]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       ".inst 0x646440f9  // bfdot z25.s, z7.h, z4.h[0]\n"
@@ -1420,15 +1420,15 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646440fb  // bfdot z27.s, z7.h, z4.h[0]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
       ".inst 0x646b40d4  // bfdot z20.s, z6.h, z3.h[1]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646c40d8  // bfdot z24.s, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x646840e9  // bfdot z9.s, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x646940ed  // bfdot z13.s, z7.h, z1.h[1]\n"
       ".inst 0x646a40f1  // bfdot z17.s, z7.h, z2.h[1]\n"
       ".inst 0x646b40f5  // bfdot z21.s, z7.h, z3.h[1]\n"
@@ -1447,15 +1447,15 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646c40fb  // bfdot z27.s, z7.h, z4.h[1]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
       ".inst 0x647340d4  // bfdot z20.s, z6.h, z3.h[2]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647440d8  // bfdot z24.s, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647040e9  // bfdot z9.s, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647140ed  // bfdot z13.s, z7.h, z1.h[2]\n"
       ".inst 0x647240f1  // bfdot z17.s, z7.h, z2.h[2]\n"
       ".inst 0x647340f5  // bfdot z21.s, z7.h, z3.h[2]\n"
@@ -1474,14 +1474,14 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647440fb  // bfdot z27.s, z7.h, z4.h[2]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
       ".inst 0x647b40d4  // bfdot z20.s, z6.h, z3.h[3]\n"
       ".inst 0x647c40d8  // bfdot z24.s, z6.h, z4.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647840e9  // bfdot z9.s, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x647940ed  // bfdot z13.s, z7.h, z1.h[3]\n"
       ".inst 0x647a40f1  // bfdot z17.s, z7.h, z2.h[3]\n"
       ".inst 0x647b40f5  // bfdot z21.s, z7.h, z3.h[3]\n"
@@ -1499,119 +1499,119 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647b40f7  // bfdot z23.s, z7.h, z3.h[3]\n"
       ".inst 0x647c40fb  // bfdot z27.s, z7.h, z4.h[3]\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 68f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 68f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -1626,36 +1626,36 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "b 70f\n"
       "69:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1683,67 +1683,67 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "71:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "73:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
+      ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
       "add x23, x23, #0x10\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x646440d8  // bfdot z24.s, z6.h, z4.h[0]\n"
       ".inst 0x646540dc  // bfdot z28.s, z6.h, z5.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
+      ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       ".inst 0x646440f9  // bfdot z25.s, z7.h, z4.h[0]\n"
       ".inst 0x646540fd  // bfdot z29.s, z7.h, z5.h[0]\n"
@@ -1847,25 +1847,25 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647d40ff  // bfdot z31.s, z7.h, z5.h[3]\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x2\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       ".inst 0x646040c8  // bfdot z8.s, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       ".inst 0x646140cc  // bfdot z12.s, z6.h, z1.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
       ".inst 0x646240d0  // bfdot z16.s, z6.h, z2.h[0]\n"
       ".inst 0x646340d4  // bfdot z20.s, z6.h, z3.h[0]\n"
       ".inst 0x646440d8  // bfdot z24.s, z6.h, z4.h[0]\n"
       ".inst 0x646540dc  // bfdot z28.s, z6.h, z5.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x646040e9  // bfdot z9.s, z7.h, z0.h[0]\n"
+      ".inst 0x646140ed  // bfdot z13.s, z7.h, z1.h[0]\n"
       ".inst 0x646240f1  // bfdot z17.s, z7.h, z2.h[0]\n"
       ".inst 0x646340f5  // bfdot z21.s, z7.h, z3.h[0]\n"
       ".inst 0x646440f9  // bfdot z25.s, z7.h, z4.h[0]\n"
@@ -1886,12 +1886,12 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646540ff  // bfdot z31.s, z7.h, z5.h[0]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x646840c8  // bfdot z8.s, z6.h, z0.h[1]\n"
       ".inst 0x646940cc  // bfdot z12.s, z6.h, z1.h[1]\n"
       ".inst 0x646a40d0  // bfdot z16.s, z6.h, z2.h[1]\n"
       ".inst 0x646b40d4  // bfdot z20.s, z6.h, z3.h[1]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x646c40d8  // bfdot z24.s, z6.h, z4.h[1]\n"
       ".inst 0x646d40dc  // bfdot z28.s, z6.h, z5.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -1917,12 +1917,12 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x646d40ff  // bfdot z31.s, z7.h, z5.h[1]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x2\n"
+      ".inst 0x647040c8  // bfdot z8.s, z6.h, z0.h[2]\n"
       ".inst 0x647140cc  // bfdot z12.s, z6.h, z1.h[2]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
       ".inst 0x647340d4  // bfdot z20.s, z6.h, z3.h[2]\n"
+      "subs x27, x27, #0x2\n"
       ".inst 0x647440d8  // bfdot z24.s, z6.h, z4.h[2]\n"
       ".inst 0x647540dc  // bfdot z28.s, z6.h, z5.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -1948,8 +1948,8 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647540ff  // bfdot z31.s, z7.h, z5.h[2]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x647840c8  // bfdot z8.s, z6.h, z0.h[3]\n"
       ".inst 0x647940cc  // bfdot z12.s, z6.h, z1.h[3]\n"
       ".inst 0x647a40d0  // bfdot z16.s, z6.h, z2.h[3]\n"
       ".inst 0x647b40d4  // bfdot z20.s, z6.h, z3.h[3]\n"
@@ -1977,115 +1977,115 @@ void sve_hybrid_bf16fp32_dot_6x4VL (
       ".inst 0x647c40fb  // bfdot z27.s, z7.h, z4.h[3]\n"
       ".inst 0x647d40ff  // bfdot z31.s, z7.h, z5.h[3]\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 71b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmin z28.s, p5/M, z28.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
-      "fmax z28.s, p5/M, z28.s, z1.s\n"
-      "fmin z29.s, p5/M, z29.s, z0.s\n"
-      "fmin z30.s, p5/M, z30.s, z0.s\n"
-      "fmin z31.s, p5/M, z31.s, z0.s\n"
-      "fmax z29.s, p5/M, z29.s, z1.s\n"
-      "fmax z30.s, p5/M, z30.s, z1.s\n"
-      "fmax z31.s, p5/M, z31.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmin z28.s, p5/M, z28.s, z1.s\n"
+      "fmin z29.s, p5/M, z29.s, z1.s\n"
+      "fmin z30.s, p5/M, z30.s, z1.s\n"
+      "fmin z31.s, p5/M, z31.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
+      "fmax z28.s, p5/M, z28.s, z0.s\n"
+      "fmax z29.s, p5/M, z29.s, z0.s\n"
+      "fmax z30.s, p5/M, z30.s, z0.s\n"
+      "fmax z31.s, p5/M, z31.s, z0.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z28.s }, p4, [x20]\n"
-      "st1w { z29.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z30.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z31.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z28.s }, p4, [x21]\n"
+      "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp
index e69293e3f1..9bb67f18d2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -103,42 +103,42 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 3f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
-      "addvl x9, x9, #4\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
@@ -156,185 +156,185 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "sub x26, x26, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "cmp x26, #0x8\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "add x26, x26, #0x10\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "subs x26, x26, #0x4\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "subs x27, x27, #0x4\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "addvl x10, x10, #8\n"
       "ble 11f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "addvl x10, x10, #8\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 6b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
       "uzp1 z9.d, z9.d, z13.d\n"
       "uzp1 z10.d, z10.d, z14.d\n"
       "uzp1 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "13:"  // Height 1: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 16f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 16f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
-      "addvl x9, x9, #4\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
@@ -350,122 +350,122 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "19:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "21:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "cmp x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "add x24, x24, #0x10\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
+      "subs x27, x27, #0x4\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
+      "addvl x10, x10, #8\n"
       "ble 24f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
       ".inst 0x6467e42b  // bfmmla z11.s, z1.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      "addvl x10, x10, #8\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 19b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
@@ -473,70 +473,70 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z7.s, p5/M, z7.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmax z7.s, p5/M, z7.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z7.s, p5/M, z7.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmax z7.s, p5/M, z7.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z7.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 29f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 29f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "mov z16.d, z8.d\n"
       "mov z20.d, z12.d\n"
       "mov z17.d, z9.d\n"
       "mov z21.d, z13.d\n"
@@ -547,29 +547,29 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
@@ -597,61 +597,58 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "34:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "cmp x26, #0x8\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -659,12 +656,15 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n"
@@ -686,29 +686,29 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
@@ -722,12 +722,12 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ble 37f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
@@ -747,17 +747,17 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 32b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
@@ -767,82 +767,82 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z18.d, z18.d, z22.d\n"
       "uzp1 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z7.s, p5/M, z7.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmax z7.s, p5/M, z7.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z7.s, p5/M, z7.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmax z7.s, p5/M, z7.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z7.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 42f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 42f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "mov z16.d, z8.d\n"
       "mov z20.d, z12.d\n"
       "mov z17.d, z9.d\n"
       "mov z21.d, z13.d\n"
@@ -853,38 +853,38 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
@@ -908,66 +908,62 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "47:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "cmp x26, #0x8\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
-      "add x22, x22, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x8\n"
       ".inst 0x6466e40e  // bfmmla z14.s, z0.h, z6.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -975,15 +971,19 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n"
@@ -1002,30 +1002,30 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
@@ -1039,12 +1039,12 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       "ble 50f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
@@ -1064,19 +1064,19 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 45b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
@@ -1089,94 +1089,94 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z22.d, z19.d, z23.d\n"
       "uzp2 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z7.s, p5/M, z7.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmax z7.s, p5/M, z7.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z7.s, p5/M, z7.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmax z7.s, p5/M, z7.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z7.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 55f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 55f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "mov z16.d, z8.d\n"
       "mov z20.d, z12.d\n"
       "mov z17.d, z9.d\n"
       "mov z21.d, z13.d\n"
@@ -1195,47 +1195,47 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
       "zip2 z23.d, z24.d, z23.d\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
@@ -1271,72 +1271,72 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "60:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "sub x26, x26, #0x8\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      "cmp x26, #0x8\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "add x24, x24, #0x10\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
-      "add x22, x22, #0x10\n"
       "trn2 z5.d, z5.d, z6.d\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
@@ -1388,25 +1388,25 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "subs x26, x26, #0x4\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
@@ -1427,8 +1427,8 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
@@ -1436,28 +1436,28 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
       "ble 63f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -1469,21 +1469,21 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
       "uzp1 z15.d, z16.d, z20.d\n"
@@ -1499,109 +1499,109 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z26.d, z26.d, z30.d\n"
       "uzp1 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z7.s, p5/M, z7.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmax z7.s, p5/M, z7.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z7.s, p5/M, z7.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmax z7.s, p5/M, z7.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z7.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 68f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 68f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "mov z16.d, z8.d\n"
       "mov z20.d, z12.d\n"
       "mov z17.d, z9.d\n"
       "mov z21.d, z13.d\n"
@@ -1620,55 +1620,55 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
       "zip2 z23.d, z24.d, z23.d\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z29.d, z26.d, z29.d\n"
       "zip1 z26.d, z27.d, z30.d\n"
       "zip2 z30.d, z27.d, z30.d\n"
@@ -1701,80 +1701,80 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "71:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "73:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
-      "cmp x26, #0x8\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      "add x24, x24, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqh { z6.h }, p0/Z, [x20]\n"
-      "add x23, x23, #0x10\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
+      "ld1rqh { z6.h }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
-      "add x20, x20, #0x10\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
+      ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e409  // bfmmla z9.s, z0.h, z7.h\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6467e451  // bfmmla z17.s, z2.h, z7.h\n"
       ".inst 0x6467e499  // bfmmla z25.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6466e49d  // bfmmla z29.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e40a  // bfmmla z10.s, z0.h, z7.h\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6467e452  // bfmmla z18.s, z2.h, z7.h\n"
       ".inst 0x6467e49a  // bfmmla z26.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
@@ -1823,26 +1823,26 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
-      "ld1h { z7.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqh { z1.h }, p0/Z, [x25]\n"
-      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z1.h }, p0/Z, [x26]\n"
+      "ld1rqh { z2.h }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z3.h }, p0/Z, [x24]\n"
+      "ld1rqh { z4.h }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqh { z4.h }, p0/Z, [x22]\n"
-      "ld1rqh { z5.h }, p0/Z, [x21]\n"
-      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
-      "ld1rqh { z6.h }, p0/Z, [x20]\n"
       "trn1 z2.d, z3.d, z4.d\n"
+      "ld1rqh { z5.h }, p0/Z, [x22]\n"
+      "ld1rqh { z6.h }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
+      "ld1h { z7.h }, p5/Z, [x10]\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e408  // bfmmla z8.s, z0.h, z7.h\n"
       ".inst 0x6467e450  // bfmmla z16.s, z2.h, z7.h\n"
       ".inst 0x6467e498  // bfmmla z24.s, z4.h, z7.h\n"
       "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       ".inst 0x6466e40c  // bfmmla z12.s, z0.h, z6.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
       ".inst 0x6466e49c  // bfmmla z28.s, z4.h, z6.h\n"
@@ -1863,8 +1863,8 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6466e49e  // bfmmla z30.s, z4.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
-      "addvl x10, x10, #8\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6467e453  // bfmmla z19.s, z2.h, z7.h\n"
       ".inst 0x6467e49b  // bfmmla z27.s, z4.h, z7.h\n"
       ".inst 0x6466e40f  // bfmmla z15.s, z0.h, z6.h\n"
@@ -1872,28 +1872,28 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e49f  // bfmmla z31.s, z4.h, z6.h\n"
       "ble 76f\n"
       "ld1h { z7.h }, p5/Z, [x10]\n"
-      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x6467e428  // bfmmla z8.s, z1.h, z7.h\n"
       ".inst 0x6467e470  // bfmmla z16.s, z3.h, z7.h\n"
       ".inst 0x6467e4b8  // bfmmla z24.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e42c  // bfmmla z12.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6466e474  // bfmmla z20.s, z3.h, z6.h\n"
       ".inst 0x6466e4bc  // bfmmla z28.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6467e429  // bfmmla z9.s, z1.h, z7.h\n"
       ".inst 0x6467e471  // bfmmla z17.s, z3.h, z7.h\n"
       ".inst 0x6467e4b9  // bfmmla z25.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e42d  // bfmmla z13.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6466e475  // bfmmla z21.s, z3.h, z6.h\n"
       ".inst 0x6466e4bd  // bfmmla z29.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6467e42a  // bfmmla z10.s, z1.h, z7.h\n"
       ".inst 0x6467e472  // bfmmla z18.s, z3.h, z7.h\n"
       ".inst 0x6467e4ba  // bfmmla z26.s, z5.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6466e476  // bfmmla z22.s, z3.h, z6.h\n"
       ".inst 0x6466e4be  // bfmmla z30.s, z5.h, z6.h\n"
       "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -1905,23 +1905,23 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       ".inst 0x6466e477  // bfmmla z23.s, z3.h, z6.h\n"
       ".inst 0x6466e4bf  // bfmmla z31.s, z5.h, z6.h\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 71b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "add x21, x22, x19, LSL #2\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x20, x21, x19, LSL #2\n"
       "uzp2 z11.d, z11.d, z15.d\n"
       "uzp1 z15.d, z16.d, z20.d\n"
       "uzp2 z16.d, z16.d, z20.d\n"
@@ -1940,104 +1940,103 @@ void sve_hybrid_bf16fp32_mmla_6x4VL (
       "uzp1 z30.d, z27.d, z31.d\n"
       "uzp2 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z7.s, p5/M, z7.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmax z7.s, p5/M, z7.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
-      "fmin z28.s, p5/M, z28.s, z0.s\n"
-      "fmin z29.s, p5/M, z29.s, z0.s\n"
-      "fmin z30.s, p5/M, z30.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z28.s, p5/M, z28.s, z1.s\n"
-      "fmax z29.s, p5/M, z29.s, z1.s\n"
-      "fmax z30.s, p5/M, z30.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z7.s, p5/M, z7.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmin z28.s, p5/M, z28.s, z1.s\n"
+      "fmin z29.s, p5/M, z29.s, z1.s\n"
+      "fmin z30.s, p5/M, z30.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmax z7.s, p5/M, z7.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
+      "fmax z28.s, p5/M, z28.s, z0.s\n"
+      "fmax z29.s, p5/M, z29.s, z0.s\n"
+      "fmax z30.s, p5/M, z30.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z7.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z23.s }, p4, [x21]\n"
-      "st1w { z28.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z29.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z30.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z23.s }, p4, [x22]\n"
+      "st1w { z28.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z29.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z30.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
-
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp
index 11f5ed2c0a..a70e66cbe4 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -102,32 +102,32 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 25f\n"
       "beq 13f\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 3f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -135,58 +135,58 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x25, x25, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x26, x26, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 11f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -196,53 +196,53 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z10.h, p4/M, z10.h, z0.h\n"
       "fmax z11.h, p4/M, z11.h, z0.h\n"
       "11:"  // Height 1: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "12:"  // Height 1: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 74f\n"
       "13:"  // Height 2
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "14:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 15f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 15f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
+      "addvl x12, x12, #4\n"
       "b 17f\n"
       "15:"  // Height 2: no bias
       "tbz %x[flags], #0, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
       "b 17f\n"
       "16:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -254,74 +254,74 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "18:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "20:"  // Height 2: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 22f\n"
       "21:"  // Height 2: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x25, x25, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x26, x26, #0x2\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "subs x26, x26, #0x1\n"
-      "add x24, x24, #0x2\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 21b\n"
       "22:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "bne 18b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
       "tbz %x[flags], #1, 23f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -339,66 +339,66 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z14.h, p4/M, z14.h, z0.h\n"
       "fmax z15.h, p4/M, z15.h, z0.h\n"
       "23:"  // Height 2: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
       "24:"  // Height 2: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 14b\n"
       "b 74f\n"
       "25:"  // Height 3
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "26:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 27f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 27f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 29f\n"
       "27:"  // Height 3: no bias
       "tbz %x[flags], #0, 28f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
       "b 29f\n"
       "28:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -414,74 +414,74 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "29:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "30:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 32f\n"
       "31:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "32:"  // Height 3: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 34f\n"
       "33:"  // Height 3: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x2\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x23, x23, #0x2\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x24, x24, #0x2\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 33b\n"
       "34:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -489,14 +489,14 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "tbz %x[flags], #1, 35f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -522,50 +522,50 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z18.h, p4/M, z18.h, z0.h\n"
       "fmax z19.h, p4/M, z19.h, z0.h\n"
       "35:"  // Height 3: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
       "36:"  // Height 3: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 26b\n"
       "b 74f\n"
       "37:"  // Height 4
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "38:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 39f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 39f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -575,26 +575,26 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "b 41f\n"
       "39:"  // Height 4: no bias
       "tbz %x[flags], #0, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
       "b 41f\n"
       "40:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -614,86 +614,86 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "41:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "42:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 43f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 44f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 44f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 44f\n"
       "43:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "44:"  // Height 4: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 46f\n"
       "45:"  // Height 4: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x2\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
+      "add x24, x24, #0x2\n"
       "add x23, x23, #0x2\n"
-      "add x22, x22, #0x2\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 45b\n"
       "46:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -703,15 +703,15 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "bne 42b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "tbz %x[flags], #1, 47f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -745,54 +745,54 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z22.h, p4/M, z22.h, z0.h\n"
       "fmax z23.h, p4/M, z23.h, z0.h\n"
       "47:"  // Height 4: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
       "48:"  // Height 4: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 38b\n"
       "b 74f\n"
       "49:"  // Height 5
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "50:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 51f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 51f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -806,31 +806,31 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "b 53f\n"
       "51:"  // Height 5: no bias
       "tbz %x[flags], #0, 52f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p3/Z, [x21]\n"
-      "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p3/Z, [x22]\n"
+      "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n"
       "b 53f\n"
       "52:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -854,98 +854,98 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "53:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "54:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 55f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 56f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 56f\n"
       "55:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "56:"  // Height 5: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 58f\n"
       "57:"  // Height 5: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
+      "add x25, x25, #0x2\n"
       "add x24, x24, #0x2\n"
-      "add x23, x23, #0x2\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x2\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
-      "add x21, x21, #0x2\n"
+      "add x22, x22, #0x2\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
       "fmla z26.h, p4/M, z6.h, z4.h\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 57b\n"
       "58:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -957,16 +957,16 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmla z23.h, p4/M, z7.h, z3.h\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "bne 54b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "tbz %x[flags], #1, 59f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -1008,61 +1008,61 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z26.h, p4/M, z26.h, z0.h\n"
       "fmax z27.h, p4/M, z27.h, z0.h\n"
       "59:"  // Height 5: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p3, [x21]\n"
-      "st1h { z25.h }, p2, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p1, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p0, [x21, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p3, [x22]\n"
+      "st1h { z25.h }, p2, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p1, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p0, [x22, #3, MUL VL]\n"
       "60:"  // Height 5: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 50b\n"
       "b 74f\n"
       "61:"  // Height 6
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0xc\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0xc\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "62:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x10\n"
-      "inch x19\n"
-      "whilelt p0.h, x19, x10\n"
-      "cbz x11, 63f\n"
-      "ld1h { z8.h }, p4/Z, [x11]\n"
-      "ld1h { z9.h }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1h { z10.h }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p0.h, x20, x11\n"
+      "cbz x12, 63f\n"
+      "ld1h { z8.h }, p4/Z, [x12]\n"
+      "ld1h { z9.h }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1h { z10.h }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1080,36 +1080,36 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "b 65f\n"
       "63:"  // Height 6: no bias
       "tbz %x[flags], #0, 64f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z8.h }, p3/Z, [x28]\n"
-      "ld1h { z9.h }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #1\n"
-      "ld1h { z10.h }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p3/Z, [x24]\n"
-      "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p3/Z, [x23]\n"
-      "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p3/Z, [x22]\n"
-      "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p3/Z, [x21]\n"
-      "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1h { z28.h }, p3/Z, [x20]\n"
-      "ld1h { z29.h }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z30.h }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z31.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #1\n"
+      "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p3/Z, [x25]\n"
+      "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p3/Z, [x24]\n"
+      "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p3/Z, [x23]\n"
+      "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p3/Z, [x22]\n"
+      "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1h { z28.h }, p3/Z, [x21]\n"
+      "ld1h { z29.h }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z30.h }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z31.h }, p0/Z, [x21, #3, MUL VL]\n"
       "b 65f\n"
       "64:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1137,110 +1137,110 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "65:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "66:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 68f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 68f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 68f\n"
       "67:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "68:"  // Height 6: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1rh { z5.h }, p4/Z, [x20]\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1rh { z5.h }, p4/Z, [x21]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 70f\n"
       "69:"  // Height 6: Multiply loop: Main loop
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x25, x25, #0x2\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x2\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
+      "add x25, x25, #0x2\n"
       "add x24, x24, #0x2\n"
-      "add x23, x23, #0x2\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z28.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x2\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x2\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
+      "add x22, x22, #0x2\n"
       "add x21, x21, #0x2\n"
-      "add x20, x20, #0x2\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
       "fmla z29.h, p4/M, z7.h, z5.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
       "fmla z22.h, p4/M, z6.h, z3.h\n"
       "fmla z26.h, p4/M, z6.h, z4.h\n"
       "fmla z30.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x9]\n"
+      "ld1h { z6.h }, p4/Z, [x10]\n"
       "fmla z11.h, p4/M, z7.h, z0.h\n"
       "fmla z15.h, p4/M, z7.h, z1.h\n"
-      "ld1rh { z0.h }, p4/Z, [x25]\n"
-      "ld1rh { z1.h }, p4/Z, [x24]\n"
+      "ld1rh { z0.h }, p4/Z, [x26]\n"
+      "ld1rh { z1.h }, p4/Z, [x25]\n"
       "fmla z19.h, p4/M, z7.h, z2.h\n"
       "fmla z23.h, p4/M, z7.h, z3.h\n"
-      "ld1rh { z2.h }, p4/Z, [x23]\n"
-      "ld1rh { z3.h }, p4/Z, [x22]\n"
+      "ld1rh { z2.h }, p4/Z, [x24]\n"
+      "ld1rh { z3.h }, p4/Z, [x23]\n"
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "fmla z31.h, p4/M, z7.h, z5.h\n"
-      "ld1rh { z4.h }, p4/Z, [x21]\n"
-      "ld1rh { z5.h }, p4/Z, [x20]\n"
-      "ld1h { z7.h }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rh { z4.h }, p4/Z, [x22]\n"
+      "ld1rh { z5.h }, p4/Z, [x21]\n"
+      "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 69b\n"
       "70:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.h, p4/M, z6.h, z0.h\n"
       "fmla z12.h, p4/M, z6.h, z1.h\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.h, p4/M, z6.h, z2.h\n"
       "fmla z20.h, p4/M, z6.h, z3.h\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "fmla z24.h, p4/M, z6.h, z4.h\n"
       "fmla z28.h, p4/M, z6.h, z5.h\n"
-      "ld1h { z6.h }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, p4/M, z7.h, z0.h\n"
       "fmla z13.h, p4/M, z7.h, z1.h\n"
       "fmla z17.h, p4/M, z7.h, z2.h\n"
       "fmla z21.h, p4/M, z7.h, z3.h\n"
       "fmla z25.h, p4/M, z7.h, z4.h\n"
       "fmla z29.h, p4/M, z7.h, z5.h\n"
-      "ld1h { z7.h }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.h, p4/M, z6.h, z0.h\n"
       "fmla z14.h, p4/M, z6.h, z1.h\n"
       "fmla z18.h, p4/M, z6.h, z2.h\n"
@@ -1254,17 +1254,17 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmla z27.h, p4/M, z7.h, z4.h\n"
       "fmla z31.h, p4/M, z7.h, z5.h\n"
       "bne 66b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "tbz %x[flags], #1, 71f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z1.h }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z0.h }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p4/Z, [x20]\n"
       "fmin z8.h, p4/M, z8.h, z1.h\n"
       "fmin z9.h, p4/M, z9.h, z1.h\n"
       "fmin z10.h, p4/M, z10.h, z1.h\n"
@@ -1314,51 +1314,51 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx (
       "fmax z30.h, p4/M, z30.h, z0.h\n"
       "fmax z31.h, p4/M, z31.h, z0.h\n"
       "71:"  // Height 6: No activation
-      "st1h { z8.h }, p3, [x28]\n"
-      "st1h { z9.h }, p2, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p1, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p3, [x24]\n"
-      "st1h { z13.h }, p2, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p1, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p0, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p3, [x23]\n"
-      "st1h { z17.h }, p2, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p1, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p0, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p3, [x22]\n"
-      "st1h { z21.h }, p2, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p1, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p0, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p3, [x21]\n"
-      "st1h { z25.h }, p2, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p1, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p0, [x21, #3, MUL VL]\n"
-      "st1h { z28.h }, p3, [x20]\n"
-      "st1h { z29.h }, p2, [x20, #1, MUL VL]\n"
-      "st1h { z30.h }, p1, [x20, #2, MUL VL]\n"
-      "st1h { z31.h }, p0, [x20, #3, MUL VL]\n"
+      "st1h { z8.h }, p3, [x9]\n"
+      "st1h { z9.h }, p2, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p1, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p3, [x25]\n"
+      "st1h { z13.h }, p2, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p1, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p0, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p3, [x24]\n"
+      "st1h { z17.h }, p2, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p1, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p0, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p3, [x23]\n"
+      "st1h { z21.h }, p2, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p1, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p0, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p3, [x22]\n"
+      "st1h { z25.h }, p2, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p1, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p0, [x22, #3, MUL VL]\n"
+      "st1h { z28.h }, p3, [x21]\n"
+      "st1h { z29.h }, p2, [x21, #1, MUL VL]\n"
+      "st1h { z30.h }, p1, [x21, #2, MUL VL]\n"
+      "st1h { z31.h }, p0, [x21, #3, MUL VL]\n"
       "72:"  // Height 6: Writeback done
-      "dech x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "dech x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 62b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 74f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "73:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "74:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp
index 09d5d8d96d..6f0b3e0008 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -102,32 +102,32 @@ void sve_hybrid_fp16_mla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 3f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -135,252 +135,252 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
-      "cmp x26, #0x8\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
-      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "fmla z10.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
-      "fmla z10.h, z6.h, z0.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x8\n"
+      "cmp x27, #0x8\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
+      "add x26, x26, #0x10\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
+      "addvl x10, x10, #4\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
       "12:"  // Height 1: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "13:"  // Height 1: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 16f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 16f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "addvl x12, x12, #4\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
       "b 18f\n"
       "17:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -392,52 +392,52 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "19:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
       "21:"  // Height 2: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "cmp x27, #0x8\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -521,216 +521,216 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z15.h, z7.h, z1.h[7]\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "fmla z13.h, z7.h, z1.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "ble 24f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.h, z6.h, z0.h[7]\n"
       "fmla z14.h, z6.h, z1.h[7]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.h, z7.h, z0.h[7]\n"
       "fmla z15.h, z7.h, z1.h[7]\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmin z12.h, p5/M, z12.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
-      "fmax z12.h, p5/M, z12.h, z1.h\n"
-      "fmin z13.h, p5/M, z13.h, z0.h\n"
-      "fmin z14.h, p5/M, z14.h, z0.h\n"
-      "fmin z15.h, p5/M, z15.h, z0.h\n"
-      "fmax z13.h, p5/M, z13.h, z1.h\n"
-      "fmax z14.h, p5/M, z14.h, z1.h\n"
-      "fmax z15.h, p5/M, z15.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmin z12.h, p5/M, z12.h, z1.h\n"
+      "fmin z13.h, p5/M, z13.h, z1.h\n"
+      "fmin z14.h, p5/M, z14.h, z1.h\n"
+      "fmin z15.h, p5/M, z15.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
+      "fmax z12.h, p5/M, z12.h, z0.h\n"
+      "fmax z13.h, p5/M, z13.h, z0.h\n"
+      "fmax z14.h, p5/M, z14.h, z0.h\n"
+      "fmax z15.h, p5/M, z15.h, z0.h\n"
       "25:"  // Height 2: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 29f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 29f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "addvl x9, x9, #4\n"
-      "mov z17.d, z9.d\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
       "b 31f\n"
       "30:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -746,63 +746,63 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "34:"  // Height 3: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "cmp x27, #0x8\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
       "fmla z14.h, z6.h, z1.h[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z18.h, z6.h, z2.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -810,64 +810,64 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z10.h, z6.h, z0.h[1]\n"
       "fmla z14.h, z6.h, z1.h[1]\n"
       "fmla z18.h, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[2]\n"
       "fmla z14.h, z6.h, z1.h[2]\n"
       "fmla z18.h, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[3]\n"
       "fmla z14.h, z6.h, z1.h[3]\n"
       "fmla z18.h, z6.h, z2.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[4]\n"
       "fmla z14.h, z6.h, z1.h[4]\n"
       "fmla z18.h, z6.h, z2.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
@@ -875,32 +875,32 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z10.h, z6.h, z0.h[5]\n"
       "fmla z14.h, z6.h, z1.h[5]\n"
       "fmla z18.h, z6.h, z2.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[6]\n"
       "fmla z14.h, z6.h, z1.h[6]\n"
       "fmla z18.h, z6.h, z2.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n"
@@ -912,19 +912,19 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[7]\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "addvl x10, x10, #4\n"
@@ -936,13 +936,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[0]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -955,13 +955,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[1]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -974,13 +974,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[2]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -993,13 +993,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[3]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1012,13 +1012,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[4]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1031,13 +1031,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[5]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1050,12 +1050,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[6]\n"
       "ble 37f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1067,116 +1067,116 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z15.h, z7.h, z1.h[7]\n"
       "fmla z19.h, z7.h, z2.h[7]\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmin z12.h, p5/M, z12.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
-      "fmax z12.h, p5/M, z12.h, z1.h\n"
-      "fmin z13.h, p5/M, z13.h, z0.h\n"
-      "fmin z14.h, p5/M, z14.h, z0.h\n"
-      "fmin z15.h, p5/M, z15.h, z0.h\n"
-      "fmin z16.h, p5/M, z16.h, z0.h\n"
-      "fmax z13.h, p5/M, z13.h, z1.h\n"
-      "fmax z14.h, p5/M, z14.h, z1.h\n"
-      "fmax z15.h, p5/M, z15.h, z1.h\n"
-      "fmax z16.h, p5/M, z16.h, z1.h\n"
-      "fmin z17.h, p5/M, z17.h, z0.h\n"
-      "fmin z18.h, p5/M, z18.h, z0.h\n"
-      "fmin z19.h, p5/M, z19.h, z0.h\n"
-      "fmax z17.h, p5/M, z17.h, z1.h\n"
-      "fmax z18.h, p5/M, z18.h, z1.h\n"
-      "fmax z19.h, p5/M, z19.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmin z12.h, p5/M, z12.h, z1.h\n"
+      "fmin z13.h, p5/M, z13.h, z1.h\n"
+      "fmin z14.h, p5/M, z14.h, z1.h\n"
+      "fmin z15.h, p5/M, z15.h, z1.h\n"
+      "fmin z16.h, p5/M, z16.h, z1.h\n"
+      "fmin z17.h, p5/M, z17.h, z1.h\n"
+      "fmin z18.h, p5/M, z18.h, z1.h\n"
+      "fmin z19.h, p5/M, z19.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
+      "fmax z12.h, p5/M, z12.h, z0.h\n"
+      "fmax z13.h, p5/M, z13.h, z0.h\n"
+      "fmax z14.h, p5/M, z14.h, z0.h\n"
+      "fmax z15.h, p5/M, z15.h, z0.h\n"
+      "fmax z16.h, p5/M, z16.h, z0.h\n"
+      "fmax z17.h, p5/M, z17.h, z0.h\n"
+      "fmax z18.h, p5/M, z18.h, z0.h\n"
+      "fmax z19.h, p5/M, z19.h, z0.h\n"
       "38:"  // Height 3: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 42f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 42f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
       "b 44f\n"
       "43:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -1196,55 +1196,55 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "47:"  // Height 4: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
-      "add x22, x22, #0x10\n"
-      "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.h, z6.h, z0.h[0]\n"
@@ -1399,21 +1399,21 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[7]\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1428,13 +1428,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[0]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
@@ -1451,13 +1451,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[1]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
@@ -1474,13 +1474,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[2]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
@@ -1497,13 +1497,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[3]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
@@ -1520,13 +1520,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[4]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
@@ -1543,13 +1543,13 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[5]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
@@ -1566,8 +1566,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[6]\n"
       "ble 50f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
@@ -1587,103 +1587,103 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z19.h, z7.h, z2.h[7]\n"
       "fmla z23.h, z7.h, z3.h[7]\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmin z12.h, p5/M, z12.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
-      "fmax z12.h, p5/M, z12.h, z1.h\n"
-      "fmin z13.h, p5/M, z13.h, z0.h\n"
-      "fmin z14.h, p5/M, z14.h, z0.h\n"
-      "fmin z15.h, p5/M, z15.h, z0.h\n"
-      "fmin z16.h, p5/M, z16.h, z0.h\n"
-      "fmax z13.h, p5/M, z13.h, z1.h\n"
-      "fmax z14.h, p5/M, z14.h, z1.h\n"
-      "fmax z15.h, p5/M, z15.h, z1.h\n"
-      "fmax z16.h, p5/M, z16.h, z1.h\n"
-      "fmin z17.h, p5/M, z17.h, z0.h\n"
-      "fmin z18.h, p5/M, z18.h, z0.h\n"
-      "fmin z19.h, p5/M, z19.h, z0.h\n"
-      "fmin z20.h, p5/M, z20.h, z0.h\n"
-      "fmax z17.h, p5/M, z17.h, z1.h\n"
-      "fmax z18.h, p5/M, z18.h, z1.h\n"
-      "fmax z19.h, p5/M, z19.h, z1.h\n"
-      "fmax z20.h, p5/M, z20.h, z1.h\n"
-      "fmin z21.h, p5/M, z21.h, z0.h\n"
-      "fmin z22.h, p5/M, z22.h, z0.h\n"
-      "fmin z23.h, p5/M, z23.h, z0.h\n"
-      "fmax z21.h, p5/M, z21.h, z1.h\n"
-      "fmax z22.h, p5/M, z22.h, z1.h\n"
-      "fmax z23.h, p5/M, z23.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmin z12.h, p5/M, z12.h, z1.h\n"
+      "fmin z13.h, p5/M, z13.h, z1.h\n"
+      "fmin z14.h, p5/M, z14.h, z1.h\n"
+      "fmin z15.h, p5/M, z15.h, z1.h\n"
+      "fmin z16.h, p5/M, z16.h, z1.h\n"
+      "fmin z17.h, p5/M, z17.h, z1.h\n"
+      "fmin z18.h, p5/M, z18.h, z1.h\n"
+      "fmin z19.h, p5/M, z19.h, z1.h\n"
+      "fmin z20.h, p5/M, z20.h, z1.h\n"
+      "fmin z21.h, p5/M, z21.h, z1.h\n"
+      "fmin z22.h, p5/M, z22.h, z1.h\n"
+      "fmin z23.h, p5/M, z23.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
+      "fmax z12.h, p5/M, z12.h, z0.h\n"
+      "fmax z13.h, p5/M, z13.h, z0.h\n"
+      "fmax z14.h, p5/M, z14.h, z0.h\n"
+      "fmax z15.h, p5/M, z15.h, z0.h\n"
+      "fmax z16.h, p5/M, z16.h, z0.h\n"
+      "fmax z17.h, p5/M, z17.h, z0.h\n"
+      "fmax z18.h, p5/M, z18.h, z0.h\n"
+      "fmax z19.h, p5/M, z19.h, z0.h\n"
+      "fmax z20.h, p5/M, z20.h, z0.h\n"
+      "fmax z21.h, p5/M, z21.h, z0.h\n"
+      "fmax z22.h, p5/M, z22.h, z0.h\n"
+      "fmax z23.h, p5/M, z23.h, z0.h\n"
       "51:"  // Height 4: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 55f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 55f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -1694,31 +1694,31 @@ void sve_hybrid_fp16_mla_6x4VL (
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p4/Z, [x21]\n"
-      "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p4/Z, [x22]\n"
+      "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
       "b 57f\n"
       "56:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -1742,61 +1742,61 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "60:"  // Height 5: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
-      "add x22, x22, #0x10\n"
-      "fmla z17.h, z7.h, z2.h[0]\n"
-      "add x21, x21, #0x10\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
+      "add x25, x25, #0x10\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
+      "fmla z17.h, z7.h, z2.h[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
       "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1805,8 +1805,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[0]\n"
       "fmla z22.h, z6.h, z3.h[0]\n"
       "fmla z26.h, z6.h, z4.h[0]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[0]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[0]\n"
       "fmla z19.h, z7.h, z2.h[0]\n"
       "fmla z23.h, z7.h, z3.h[0]\n"
@@ -1817,8 +1817,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
@@ -1830,8 +1830,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[1]\n"
       "fmla z22.h, z6.h, z3.h[1]\n"
       "fmla z26.h, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[1]\n"
       "fmla z19.h, z7.h, z2.h[1]\n"
       "fmla z23.h, z7.h, z3.h[1]\n"
@@ -1842,8 +1842,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
@@ -1854,8 +1854,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[2]\n"
       "fmla z22.h, z6.h, z3.h[2]\n"
       "fmla z26.h, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[2]\n"
       "fmla z19.h, z7.h, z2.h[2]\n"
       "fmla z23.h, z7.h, z3.h[2]\n"
@@ -1866,8 +1866,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
@@ -1878,8 +1878,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[3]\n"
       "fmla z22.h, z6.h, z3.h[3]\n"
       "fmla z26.h, z6.h, z4.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z11.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "fmla z15.h, z7.h, z1.h[3]\n"
       "fmla z19.h, z7.h, z2.h[3]\n"
       "fmla z23.h, z7.h, z3.h[3]\n"
@@ -1890,8 +1890,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
@@ -1902,8 +1902,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[4]\n"
       "fmla z22.h, z6.h, z3.h[4]\n"
       "fmla z26.h, z6.h, z4.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[4]\n"
       "fmla z19.h, z7.h, z2.h[4]\n"
       "fmla z23.h, z7.h, z3.h[4]\n"
@@ -1914,8 +1914,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
@@ -1927,8 +1927,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[5]\n"
       "fmla z22.h, z6.h, z3.h[5]\n"
       "fmla z26.h, z6.h, z4.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[5]\n"
       "fmla z19.h, z7.h, z2.h[5]\n"
       "fmla z23.h, z7.h, z3.h[5]\n"
@@ -1939,8 +1939,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
@@ -1951,8 +1951,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z18.h, z6.h, z2.h[6]\n"
       "fmla z22.h, z6.h, z3.h[6]\n"
       "fmla z26.h, z6.h, z4.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.h, z7.h, z1.h[6]\n"
       "fmla z19.h, z7.h, z2.h[6]\n"
       "fmla z23.h, z7.h, z3.h[6]\n"
@@ -1963,8 +1963,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
@@ -1982,23 +1982,23 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[7]\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
-      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
@@ -2016,15 +2016,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[0]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[1]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[1]\n"
       "fmla z17.h, z7.h, z2.h[1]\n"
       "fmla z21.h, z7.h, z3.h[1]\n"
@@ -2043,15 +2043,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[1]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[2]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[2]\n"
       "fmla z17.h, z7.h, z2.h[2]\n"
       "fmla z21.h, z7.h, z3.h[2]\n"
@@ -2070,15 +2070,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[2]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[3]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[3]\n"
       "fmla z17.h, z7.h, z2.h[3]\n"
       "fmla z21.h, z7.h, z3.h[3]\n"
@@ -2097,15 +2097,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[3]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[4]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[4]\n"
       "fmla z17.h, z7.h, z2.h[4]\n"
       "fmla z21.h, z7.h, z3.h[4]\n"
@@ -2124,15 +2124,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[4]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[5]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[5]\n"
       "fmla z17.h, z7.h, z2.h[5]\n"
       "fmla z21.h, z7.h, z3.h[5]\n"
@@ -2151,15 +2151,15 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[5]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[6]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[6]\n"
       "fmla z17.h, z7.h, z2.h[6]\n"
       "fmla z21.h, z7.h, z3.h[6]\n"
@@ -2178,14 +2178,14 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[6]\n"
       "ble 63f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
       "fmla z24.h, z6.h, z4.h[7]\n"
-      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.h, z7.h, z0.h[7]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.h, z7.h, z1.h[7]\n"
       "fmla z17.h, z7.h, z2.h[7]\n"
       "fmla z21.h, z7.h, z3.h[7]\n"
@@ -2203,119 +2203,119 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z23.h, z7.h, z3.h[7]\n"
       "fmla z27.h, z7.h, z4.h[7]\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmin z12.h, p5/M, z12.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
-      "fmax z12.h, p5/M, z12.h, z1.h\n"
-      "fmin z13.h, p5/M, z13.h, z0.h\n"
-      "fmin z14.h, p5/M, z14.h, z0.h\n"
-      "fmin z15.h, p5/M, z15.h, z0.h\n"
-      "fmin z16.h, p5/M, z16.h, z0.h\n"
-      "fmax z13.h, p5/M, z13.h, z1.h\n"
-      "fmax z14.h, p5/M, z14.h, z1.h\n"
-      "fmax z15.h, p5/M, z15.h, z1.h\n"
-      "fmax z16.h, p5/M, z16.h, z1.h\n"
-      "fmin z17.h, p5/M, z17.h, z0.h\n"
-      "fmin z18.h, p5/M, z18.h, z0.h\n"
-      "fmin z19.h, p5/M, z19.h, z0.h\n"
-      "fmin z20.h, p5/M, z20.h, z0.h\n"
-      "fmax z17.h, p5/M, z17.h, z1.h\n"
-      "fmax z18.h, p5/M, z18.h, z1.h\n"
-      "fmax z19.h, p5/M, z19.h, z1.h\n"
-      "fmax z20.h, p5/M, z20.h, z1.h\n"
-      "fmin z21.h, p5/M, z21.h, z0.h\n"
-      "fmin z22.h, p5/M, z22.h, z0.h\n"
-      "fmin z23.h, p5/M, z23.h, z0.h\n"
-      "fmin z24.h, p5/M, z24.h, z0.h\n"
-      "fmax z21.h, p5/M, z21.h, z1.h\n"
-      "fmax z22.h, p5/M, z22.h, z1.h\n"
-      "fmax z23.h, p5/M, z23.h, z1.h\n"
-      "fmax z24.h, p5/M, z24.h, z1.h\n"
-      "fmin z25.h, p5/M, z25.h, z0.h\n"
-      "fmin z26.h, p5/M, z26.h, z0.h\n"
-      "fmin z27.h, p5/M, z27.h, z0.h\n"
-      "fmax z25.h, p5/M, z25.h, z1.h\n"
-      "fmax z26.h, p5/M, z26.h, z1.h\n"
-      "fmax z27.h, p5/M, z27.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmin z12.h, p5/M, z12.h, z1.h\n"
+      "fmin z13.h, p5/M, z13.h, z1.h\n"
+      "fmin z14.h, p5/M, z14.h, z1.h\n"
+      "fmin z15.h, p5/M, z15.h, z1.h\n"
+      "fmin z16.h, p5/M, z16.h, z1.h\n"
+      "fmin z17.h, p5/M, z17.h, z1.h\n"
+      "fmin z18.h, p5/M, z18.h, z1.h\n"
+      "fmin z19.h, p5/M, z19.h, z1.h\n"
+      "fmin z20.h, p5/M, z20.h, z1.h\n"
+      "fmin z21.h, p5/M, z21.h, z1.h\n"
+      "fmin z22.h, p5/M, z22.h, z1.h\n"
+      "fmin z23.h, p5/M, z23.h, z1.h\n"
+      "fmin z24.h, p5/M, z24.h, z1.h\n"
+      "fmin z25.h, p5/M, z25.h, z1.h\n"
+      "fmin z26.h, p5/M, z26.h, z1.h\n"
+      "fmin z27.h, p5/M, z27.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
+      "fmax z12.h, p5/M, z12.h, z0.h\n"
+      "fmax z13.h, p5/M, z13.h, z0.h\n"
+      "fmax z14.h, p5/M, z14.h, z0.h\n"
+      "fmax z15.h, p5/M, z15.h, z0.h\n"
+      "fmax z16.h, p5/M, z16.h, z0.h\n"
+      "fmax z17.h, p5/M, z17.h, z0.h\n"
+      "fmax z18.h, p5/M, z18.h, z0.h\n"
+      "fmax z19.h, p5/M, z19.h, z0.h\n"
+      "fmax z20.h, p5/M, z20.h, z0.h\n"
+      "fmax z21.h, p5/M, z21.h, z0.h\n"
+      "fmax z22.h, p5/M, z22.h, z0.h\n"
+      "fmax z23.h, p5/M, z23.h, z0.h\n"
+      "fmax z24.h, p5/M, z24.h, z0.h\n"
+      "fmax z25.h, p5/M, z25.h, z0.h\n"
+      "fmax z26.h, p5/M, z26.h, z0.h\n"
+      "fmax z27.h, p5/M, z27.h, z0.h\n"
       "64:"  // Height 5: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p4, [x21]\n"
-      "st1h { z25.h }, p3, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p2, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p1, [x21, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p4, [x22]\n"
+      "st1h { z25.h }, p3, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p2, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p1, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0xc\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0xc\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p3.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p2.h, x19, x11\n"
-      "inch x19\n"
-      "whilelt p1.h, x19, x11\n"
-      "cbz x9, 68f\n"
-      "ld1h { z8.h }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p3.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p2.h, x20, x11\n"
+      "inch x20\n"
+      "whilelt p1.h, x20, x11\n"
+      "cbz x12, 68f\n"
+      "ld1h { z8.h }, p5/Z, [x12]\n"
+      "ld1h { z9.h }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1h { z9.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1h { z10.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1h { z11.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1h { z10.h }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1h { z11.h }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -2330,36 +2330,36 @@ void sve_hybrid_fp16_mla_6x4VL (
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1h { z8.h }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "ld1h { z9.h }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1h { z10.h }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #1\n"
-      "ld1h { z11.h }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #1\n"
-      "ld1h { z12.h }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #1\n"
-      "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #1\n"
-      "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1h { z16.h }, p4/Z, [x23]\n"
-      "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1h { z20.h }, p4/Z, [x22]\n"
-      "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1h { z24.h }, p4/Z, [x21]\n"
-      "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1h { z28.h }, p4/Z, [x20]\n"
-      "ld1h { z29.h }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z30.h }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z31.h }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "ld1h { z8.h }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #1\n"
+      "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z12.h }, p4/Z, [x25]\n"
+      "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1h { z16.h }, p4/Z, [x24]\n"
+      "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1h { z20.h }, p4/Z, [x23]\n"
+      "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1h { z24.h }, p4/Z, [x22]\n"
+      "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1h { z28.h }, p4/Z, [x21]\n"
+      "ld1h { z29.h }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1h { z30.h }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1h { z31.h }, p1/Z, [x21, #3, MUL VL]\n"
       "b 70f\n"
       "69:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -2387,67 +2387,67 @@ void sve_hybrid_fp16_mla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "71:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #1\n"
-      "add x24, x24, x19, LSL #1\n"
-      "add x23, x23, x19, LSL #1\n"
-      "add x22, x22, x19, LSL #1\n"
-      "add x21, x21, x19, LSL #1\n"
-      "add x20, x20, x19, LSL #1\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #1\n"
+      "add x25, x25, x20, LSL #1\n"
+      "add x24, x24, x20, LSL #1\n"
+      "add x23, x23, x20, LSL #1\n"
+      "add x22, x22, x20, LSL #1\n"
+      "add x21, x21, x20, LSL #1\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "73:"  // Height 6: input setup done
-      "cmp x26, #0x8\n"
+      "cmp x27, #0x8\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x8\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "cmp x27, #0x8\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x8\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "cmp x26, #0x8\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
+      "fmla z20.h, z6.h, z3.h[0]\n"
       "add x23, x23, #0x10\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      "fmla z20.h, z6.h, z3.h[0]\n"
-      "add x21, x21, #0x10\n"
-      "fmla z17.h, z7.h, z2.h[0]\n"
-      "add x20, x20, #0x10\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z28.h, z6.h, z5.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
+      "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
       "fmla z29.h, z7.h, z5.h[0]\n"
@@ -2664,25 +2664,25 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[7]\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
+      "whilelt p0.h, XZR, x27\n"
+      "ld1rqh { z0.h }, p0/Z, [x26]\n"
+      "ld1rqh { z1.h }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqh { z2.h }, p0/Z, [x24]\n"
+      "ld1rqh { z3.h }, p0/Z, [x23]\n"
+      "ld1rqh { z4.h }, p0/Z, [x22]\n"
+      "ld1rqh { z5.h }, p0/Z, [x21]\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "whilelt p0.h, XZR, x26\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqh { z0.h }, p0/Z, [x25]\n"
       "fmla z8.h, z6.h, z0.h[0]\n"
-      "ld1rqh { z1.h }, p0/Z, [x24]\n"
-      "fmla z9.h, z7.h, z0.h[0]\n"
-      "ld1rqh { z2.h }, p0/Z, [x23]\n"
-      "ld1rqh { z3.h }, p0/Z, [x22]\n"
       "fmla z12.h, z6.h, z1.h[0]\n"
-      "ld1rqh { z4.h }, p0/Z, [x21]\n"
-      "fmla z13.h, z7.h, z1.h[0]\n"
-      "ld1rqh { z5.h }, p0/Z, [x20]\n"
       "fmla z16.h, z6.h, z2.h[0]\n"
       "fmla z20.h, z6.h, z3.h[0]\n"
       "fmla z24.h, z6.h, z4.h[0]\n"
       "fmla z28.h, z6.h, z5.h[0]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.h, z7.h, z0.h[0]\n"
+      "fmla z13.h, z7.h, z1.h[0]\n"
       "fmla z17.h, z7.h, z2.h[0]\n"
       "fmla z21.h, z7.h, z3.h[0]\n"
       "fmla z25.h, z7.h, z4.h[0]\n"
@@ -2703,12 +2703,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[0]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[1]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[1]\n"
       "fmla z12.h, z6.h, z1.h[1]\n"
       "fmla z16.h, z6.h, z2.h[1]\n"
       "fmla z20.h, z6.h, z3.h[1]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[1]\n"
       "fmla z28.h, z6.h, z5.h[1]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2734,12 +2734,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[1]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[2]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[2]\n"
       "fmla z12.h, z6.h, z1.h[2]\n"
       "fmla z16.h, z6.h, z2.h[2]\n"
       "fmla z20.h, z6.h, z3.h[2]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[2]\n"
       "fmla z28.h, z6.h, z5.h[2]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2765,12 +2765,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[2]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[3]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[3]\n"
       "fmla z12.h, z6.h, z1.h[3]\n"
       "fmla z16.h, z6.h, z2.h[3]\n"
       "fmla z20.h, z6.h, z3.h[3]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[3]\n"
       "fmla z28.h, z6.h, z5.h[3]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2796,12 +2796,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[3]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[4]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[4]\n"
       "fmla z12.h, z6.h, z1.h[4]\n"
       "fmla z16.h, z6.h, z2.h[4]\n"
       "fmla z20.h, z6.h, z3.h[4]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[4]\n"
       "fmla z28.h, z6.h, z5.h[4]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2827,12 +2827,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[4]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[5]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[5]\n"
       "fmla z12.h, z6.h, z1.h[5]\n"
       "fmla z16.h, z6.h, z2.h[5]\n"
       "fmla z20.h, z6.h, z3.h[5]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[5]\n"
       "fmla z28.h, z6.h, z5.h[5]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2858,12 +2858,12 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[5]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[6]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.h, z6.h, z0.h[6]\n"
       "fmla z12.h, z6.h, z1.h[6]\n"
       "fmla z16.h, z6.h, z2.h[6]\n"
       "fmla z20.h, z6.h, z3.h[6]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.h, z6.h, z4.h[6]\n"
       "fmla z28.h, z6.h, z5.h[6]\n"
       "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
@@ -2889,8 +2889,8 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z31.h, z7.h, z5.h[6]\n"
       "ble 76f\n"
       "ld1h { z6.h }, p5/Z, [x10]\n"
-      "fmla z8.h, z6.h, z0.h[7]\n"
       "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.h, z6.h, z0.h[7]\n"
       "fmla z12.h, z6.h, z1.h[7]\n"
       "fmla z16.h, z6.h, z2.h[7]\n"
       "fmla z20.h, z6.h, z3.h[7]\n"
@@ -2918,115 +2918,115 @@ void sve_hybrid_fp16_mla_6x4VL (
       "fmla z27.h, z7.h, z4.h[7]\n"
       "fmla z31.h, z7.h, z5.h[7]\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 71b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #1\n"
-      "add x23, x24, x19, LSL #1\n"
-      "add x22, x23, x19, LSL #1\n"
-      "add x21, x22, x19, LSL #1\n"
-      "add x20, x21, x19, LSL #1\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #1\n"
+      "add x24, x25, x20, LSL #1\n"
+      "add x23, x24, x20, LSL #1\n"
+      "add x22, x23, x20, LSL #1\n"
+      "add x21, x22, x20, LSL #1\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rh { z1.h }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rh { z0.h }, p5/Z, [x19]\n"
-      "fmin z8.h, p5/M, z8.h, z0.h\n"
-      "fmin z9.h, p5/M, z9.h, z0.h\n"
-      "fmin z10.h, p5/M, z10.h, z0.h\n"
-      "fmin z11.h, p5/M, z11.h, z0.h\n"
-      "fmin z12.h, p5/M, z12.h, z0.h\n"
-      "fmax z8.h, p5/M, z8.h, z1.h\n"
-      "fmax z9.h, p5/M, z9.h, z1.h\n"
-      "fmax z10.h, p5/M, z10.h, z1.h\n"
-      "fmax z11.h, p5/M, z11.h, z1.h\n"
-      "fmax z12.h, p5/M, z12.h, z1.h\n"
-      "fmin z13.h, p5/M, z13.h, z0.h\n"
-      "fmin z14.h, p5/M, z14.h, z0.h\n"
-      "fmin z15.h, p5/M, z15.h, z0.h\n"
-      "fmin z16.h, p5/M, z16.h, z0.h\n"
-      "fmax z13.h, p5/M, z13.h, z1.h\n"
-      "fmax z14.h, p5/M, z14.h, z1.h\n"
-      "fmax z15.h, p5/M, z15.h, z1.h\n"
-      "fmax z16.h, p5/M, z16.h, z1.h\n"
-      "fmin z17.h, p5/M, z17.h, z0.h\n"
-      "fmin z18.h, p5/M, z18.h, z0.h\n"
-      "fmin z19.h, p5/M, z19.h, z0.h\n"
-      "fmin z20.h, p5/M, z20.h, z0.h\n"
-      "fmax z17.h, p5/M, z17.h, z1.h\n"
-      "fmax z18.h, p5/M, z18.h, z1.h\n"
-      "fmax z19.h, p5/M, z19.h, z1.h\n"
-      "fmax z20.h, p5/M, z20.h, z1.h\n"
-      "fmin z21.h, p5/M, z21.h, z0.h\n"
-      "fmin z22.h, p5/M, z22.h, z0.h\n"
-      "fmin z23.h, p5/M, z23.h, z0.h\n"
-      "fmin z24.h, p5/M, z24.h, z0.h\n"
-      "fmax z21.h, p5/M, z21.h, z1.h\n"
-      "fmax z22.h, p5/M, z22.h, z1.h\n"
-      "fmax z23.h, p5/M, z23.h, z1.h\n"
-      "fmax z24.h, p5/M, z24.h, z1.h\n"
-      "fmin z25.h, p5/M, z25.h, z0.h\n"
-      "fmin z26.h, p5/M, z26.h, z0.h\n"
-      "fmin z27.h, p5/M, z27.h, z0.h\n"
-      "fmin z28.h, p5/M, z28.h, z0.h\n"
-      "fmax z25.h, p5/M, z25.h, z1.h\n"
-      "fmax z26.h, p5/M, z26.h, z1.h\n"
-      "fmax z27.h, p5/M, z27.h, z1.h\n"
-      "fmax z28.h, p5/M, z28.h, z1.h\n"
-      "fmin z29.h, p5/M, z29.h, z0.h\n"
-      "fmin z30.h, p5/M, z30.h, z0.h\n"
-      "fmin z31.h, p5/M, z31.h, z0.h\n"
-      "fmax z29.h, p5/M, z29.h, z1.h\n"
-      "fmax z30.h, p5/M, z30.h, z1.h\n"
-      "fmax z31.h, p5/M, z31.h, z1.h\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rh { z1.h }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rh { z0.h }, p5/Z, [x20]\n"
+      "fmin z8.h, p5/M, z8.h, z1.h\n"
+      "fmin z9.h, p5/M, z9.h, z1.h\n"
+      "fmin z10.h, p5/M, z10.h, z1.h\n"
+      "fmin z11.h, p5/M, z11.h, z1.h\n"
+      "fmin z12.h, p5/M, z12.h, z1.h\n"
+      "fmin z13.h, p5/M, z13.h, z1.h\n"
+      "fmin z14.h, p5/M, z14.h, z1.h\n"
+      "fmin z15.h, p5/M, z15.h, z1.h\n"
+      "fmin z16.h, p5/M, z16.h, z1.h\n"
+      "fmin z17.h, p5/M, z17.h, z1.h\n"
+      "fmin z18.h, p5/M, z18.h, z1.h\n"
+      "fmin z19.h, p5/M, z19.h, z1.h\n"
+      "fmin z20.h, p5/M, z20.h, z1.h\n"
+      "fmin z21.h, p5/M, z21.h, z1.h\n"
+      "fmin z22.h, p5/M, z22.h, z1.h\n"
+      "fmin z23.h, p5/M, z23.h, z1.h\n"
+      "fmin z24.h, p5/M, z24.h, z1.h\n"
+      "fmin z25.h, p5/M, z25.h, z1.h\n"
+      "fmin z26.h, p5/M, z26.h, z1.h\n"
+      "fmin z27.h, p5/M, z27.h, z1.h\n"
+      "fmin z28.h, p5/M, z28.h, z1.h\n"
+      "fmin z29.h, p5/M, z29.h, z1.h\n"
+      "fmin z30.h, p5/M, z30.h, z1.h\n"
+      "fmin z31.h, p5/M, z31.h, z1.h\n"
+      "fmax z8.h, p5/M, z8.h, z0.h\n"
+      "fmax z9.h, p5/M, z9.h, z0.h\n"
+      "fmax z10.h, p5/M, z10.h, z0.h\n"
+      "fmax z11.h, p5/M, z11.h, z0.h\n"
+      "fmax z12.h, p5/M, z12.h, z0.h\n"
+      "fmax z13.h, p5/M, z13.h, z0.h\n"
+      "fmax z14.h, p5/M, z14.h, z0.h\n"
+      "fmax z15.h, p5/M, z15.h, z0.h\n"
+      "fmax z16.h, p5/M, z16.h, z0.h\n"
+      "fmax z17.h, p5/M, z17.h, z0.h\n"
+      "fmax z18.h, p5/M, z18.h, z0.h\n"
+      "fmax z19.h, p5/M, z19.h, z0.h\n"
+      "fmax z20.h, p5/M, z20.h, z0.h\n"
+      "fmax z21.h, p5/M, z21.h, z0.h\n"
+      "fmax z22.h, p5/M, z22.h, z0.h\n"
+      "fmax z23.h, p5/M, z23.h, z0.h\n"
+      "fmax z24.h, p5/M, z24.h, z0.h\n"
+      "fmax z25.h, p5/M, z25.h, z0.h\n"
+      "fmax z26.h, p5/M, z26.h, z0.h\n"
+      "fmax z27.h, p5/M, z27.h, z0.h\n"
+      "fmax z28.h, p5/M, z28.h, z0.h\n"
+      "fmax z29.h, p5/M, z29.h, z0.h\n"
+      "fmax z30.h, p5/M, z30.h, z0.h\n"
+      "fmax z31.h, p5/M, z31.h, z0.h\n"
       "77:"  // Height 6: No activation
-      "st1h { z8.h }, p4, [x28]\n"
-      "st1h { z9.h }, p3, [x28, #1, MUL VL]\n"
-      "st1h { z10.h }, p2, [x28, #2, MUL VL]\n"
-      "st1h { z11.h }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1h { z12.h }, p4, [x24]\n"
-      "st1h { z13.h }, p3, [x24, #1, MUL VL]\n"
-      "st1h { z14.h }, p2, [x24, #2, MUL VL]\n"
-      "st1h { z15.h }, p1, [x24, #3, MUL VL]\n"
-      "st1h { z16.h }, p4, [x23]\n"
-      "st1h { z17.h }, p3, [x23, #1, MUL VL]\n"
-      "st1h { z18.h }, p2, [x23, #2, MUL VL]\n"
-      "st1h { z19.h }, p1, [x23, #3, MUL VL]\n"
-      "st1h { z20.h }, p4, [x22]\n"
-      "st1h { z21.h }, p3, [x22, #1, MUL VL]\n"
-      "st1h { z22.h }, p2, [x22, #2, MUL VL]\n"
-      "st1h { z23.h }, p1, [x22, #3, MUL VL]\n"
-      "st1h { z24.h }, p4, [x21]\n"
-      "st1h { z25.h }, p3, [x21, #1, MUL VL]\n"
-      "st1h { z26.h }, p2, [x21, #2, MUL VL]\n"
-      "st1h { z27.h }, p1, [x21, #3, MUL VL]\n"
-      "st1h { z28.h }, p4, [x20]\n"
-      "st1h { z29.h }, p3, [x20, #1, MUL VL]\n"
-      "st1h { z30.h }, p2, [x20, #2, MUL VL]\n"
-      "st1h { z31.h }, p1, [x20, #3, MUL VL]\n"
+      "st1h { z8.h }, p4, [x9]\n"
+      "st1h { z9.h }, p3, [x9, #1, MUL VL]\n"
+      "st1h { z10.h }, p2, [x9, #2, MUL VL]\n"
+      "st1h { z11.h }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1h { z12.h }, p4, [x25]\n"
+      "st1h { z13.h }, p3, [x25, #1, MUL VL]\n"
+      "st1h { z14.h }, p2, [x25, #2, MUL VL]\n"
+      "st1h { z15.h }, p1, [x25, #3, MUL VL]\n"
+      "st1h { z16.h }, p4, [x24]\n"
+      "st1h { z17.h }, p3, [x24, #1, MUL VL]\n"
+      "st1h { z18.h }, p2, [x24, #2, MUL VL]\n"
+      "st1h { z19.h }, p1, [x24, #3, MUL VL]\n"
+      "st1h { z20.h }, p4, [x23]\n"
+      "st1h { z21.h }, p3, [x23, #1, MUL VL]\n"
+      "st1h { z22.h }, p2, [x23, #2, MUL VL]\n"
+      "st1h { z23.h }, p1, [x23, #3, MUL VL]\n"
+      "st1h { z24.h }, p4, [x22]\n"
+      "st1h { z25.h }, p3, [x22, #1, MUL VL]\n"
+      "st1h { z26.h }, p2, [x22, #2, MUL VL]\n"
+      "st1h { z27.h }, p1, [x22, #3, MUL VL]\n"
+      "st1h { z28.h }, p4, [x21]\n"
+      "st1h { z29.h }, p3, [x21, #1, MUL VL]\n"
+      "st1h { z30.h }, p2, [x21, #2, MUL VL]\n"
+      "st1h { z31.h }, p1, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
       "dech x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0xc\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0xc\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp
index 30b6a54277..9ae51af59b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -102,32 +102,32 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 25f\n"
       "beq 13f\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 3f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -135,58 +135,58 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x25, x25, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x26, x26, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 11f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -196,53 +196,53 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z10.s, p4/M, z10.s, z0.s\n"
       "fmax z11.s, p4/M, z11.s, z0.s\n"
       "11:"  // Height 1: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "12:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 74f\n"
       "13:"  // Height 2
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "14:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 15f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 15f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
+      "addvl x12, x12, #4\n"
       "b 17f\n"
       "15:"  // Height 2: no bias
       "tbz %x[flags], #0, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
       "b 17f\n"
       "16:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -254,74 +254,74 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "18:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "20:"  // Height 2: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 22f\n"
       "21:"  // Height 2: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x25, x25, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x26, x26, #0x4\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "subs x26, x26, #0x1\n"
-      "add x24, x24, #0x4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
+      "add x25, x25, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 21b\n"
       "22:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "bne 18b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "tbz %x[flags], #1, 23f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -339,66 +339,66 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z14.s, p4/M, z14.s, z0.s\n"
       "fmax z15.s, p4/M, z15.s, z0.s\n"
       "23:"  // Height 2: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
       "24:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 14b\n"
       "b 74f\n"
       "25:"  // Height 3
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "26:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 27f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 27f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 29f\n"
       "27:"  // Height 3: no bias
       "tbz %x[flags], #0, 28f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
       "b 29f\n"
       "28:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -414,74 +414,74 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "29:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "30:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 32f\n"
       "31:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "32:"  // Height 3: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 34f\n"
       "33:"  // Height 3: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x23, x23, #0x4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x24, x24, #0x4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 33b\n"
       "34:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -489,14 +489,14 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 35f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -522,50 +522,50 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z18.s, p4/M, z18.s, z0.s\n"
       "fmax z19.s, p4/M, z19.s, z0.s\n"
       "35:"  // Height 3: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
       "36:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 26b\n"
       "b 74f\n"
       "37:"  // Height 4
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "38:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 39f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 39f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -575,26 +575,26 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "b 41f\n"
       "39:"  // Height 4: no bias
       "tbz %x[flags], #0, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
       "b 41f\n"
       "40:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -614,86 +614,86 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "41:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "42:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 43f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 44f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 44f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 44f\n"
       "43:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "44:"  // Height 4: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 46f\n"
       "45:"  // Height 4: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 45b\n"
       "46:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -703,15 +703,15 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "bne 42b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 47f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -745,54 +745,54 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z22.s, p4/M, z22.s, z0.s\n"
       "fmax z23.s, p4/M, z23.s, z0.s\n"
       "47:"  // Height 4: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
       "48:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 38b\n"
       "b 74f\n"
       "49:"  // Height 5
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "50:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 51f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 51f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -806,31 +806,31 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "b 53f\n"
       "51:"  // Height 5: no bias
       "tbz %x[flags], #0, 52f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n"
       "b 53f\n"
       "52:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -854,98 +854,98 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "53:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "54:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 55f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 56f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 56f\n"
       "55:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "56:"  // Height 5: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 58f\n"
       "57:"  // Height 5: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
-      "add x21, x21, #0x4\n"
+      "add x22, x22, #0x4\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
       "fmla z26.s, p4/M, z6.s, z4.s\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 57b\n"
       "58:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -957,16 +957,16 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmla z23.s, p4/M, z7.s, z3.s\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "bne 54b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 59f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -1008,61 +1008,61 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z26.s, p4/M, z26.s, z0.s\n"
       "fmax z27.s, p4/M, z27.s, z0.s\n"
       "59:"  // Height 5: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x21]\n"
-      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x22]\n"
+      "st1w { z25.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x22, #3, MUL VL]\n"
       "60:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 50b\n"
       "b 74f\n"
       "61:"  // Height 6
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "62:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
-      "cbz x11, 63f\n"
-      "ld1w { z8.s }, p4/Z, [x11]\n"
-      "ld1w { z9.s }, p4/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p4/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
+      "cbz x12, 63f\n"
+      "ld1w { z8.s }, p4/Z, [x12]\n"
+      "ld1w { z9.s }, p4/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p4/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p4/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
-      "addvl x11, x11, #4\n"
       "mov z16.d, z8.d\n"
       "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "mov z20.d, z8.d\n"
@@ -1080,36 +1080,36 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "b 65f\n"
       "63:"  // Height 6: no bias
       "tbz %x[flags], #0, 64f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x24]\n"
-      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x23]\n"
-      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x22]\n"
-      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21]\n"
-      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p3/Z, [x20]\n"
-      "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x25]\n"
+      "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x24]\n"
+      "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x23]\n"
+      "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p3/Z, [x21]\n"
+      "ld1w { z29.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z31.s }, p0/Z, [x21, #3, MUL VL]\n"
       "b 65f\n"
       "64:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1137,110 +1137,110 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "65:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "66:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 68f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 68f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 68f\n"
       "67:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "68:"  // Height 6: input setup done
-      "subs x26, x26, #0x1\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 70f\n"
       "69:"  // Height 6: Multiply loop: Main loop
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x1\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z28.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
+      "add x22, x22, #0x4\n"
       "add x21, x21, #0x4\n"
-      "add x20, x20, #0x4\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
       "fmla z29.s, p4/M, z7.s, z5.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
       "fmla z22.s, p4/M, z6.s, z3.s\n"
       "fmla z26.s, p4/M, z6.s, z4.s\n"
       "fmla z30.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x9]\n"
+      "ld1w { z6.s }, p4/Z, [x10]\n"
       "fmla z11.s, p4/M, z7.s, z0.s\n"
       "fmla z15.s, p4/M, z7.s, z1.s\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "fmla z19.s, p4/M, z7.s, z2.s\n"
       "fmla z23.s, p4/M, z7.s, z3.s\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "fmla z31.s, p4/M, z7.s, z5.s\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1w { z7.s }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 69b\n"
       "70:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "fmla z8.s, p4/M, z6.s, z0.s\n"
       "fmla z12.s, p4/M, z6.s, z1.s\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "fmla z16.s, p4/M, z6.s, z2.s\n"
       "fmla z20.s, p4/M, z6.s, z3.s\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "fmla z24.s, p4/M, z6.s, z4.s\n"
       "fmla z28.s, p4/M, z6.s, z5.s\n"
-      "ld1w { z6.s }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, p4/M, z7.s, z0.s\n"
       "fmla z13.s, p4/M, z7.s, z1.s\n"
       "fmla z17.s, p4/M, z7.s, z2.s\n"
       "fmla z21.s, p4/M, z7.s, z3.s\n"
       "fmla z25.s, p4/M, z7.s, z4.s\n"
       "fmla z29.s, p4/M, z7.s, z5.s\n"
-      "ld1w { z7.s }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "fmla z10.s, p4/M, z6.s, z0.s\n"
       "fmla z14.s, p4/M, z6.s, z1.s\n"
       "fmla z18.s, p4/M, z6.s, z2.s\n"
@@ -1254,17 +1254,17 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmla z27.s, p4/M, z7.s, z4.s\n"
       "fmla z31.s, p4/M, z7.s, z5.s\n"
       "bne 66b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 71f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p4/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p4/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p4/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p4/Z, [x20]\n"
       "fmin z8.s, p4/M, z8.s, z1.s\n"
       "fmin z9.s, p4/M, z9.s, z1.s\n"
       "fmin z10.s, p4/M, z10.s, z1.s\n"
@@ -1314,51 +1314,51 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx (
       "fmax z30.s, p4/M, z30.s, z0.s\n"
       "fmax z31.s, p4/M, z31.s, z0.s\n"
       "71:"  // Height 6: No activation
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x24]\n"
-      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x23]\n"
-      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x22]\n"
-      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x21]\n"
-      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z28.s }, p3, [x20]\n"
-      "st1w { z29.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z30.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z31.s }, p0, [x20, #3, MUL VL]\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x25]\n"
+      "st1w { z13.s }, p2, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x24]\n"
+      "st1w { z17.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23]\n"
+      "st1w { z21.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x22]\n"
+      "st1w { z25.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z28.s }, p3, [x21]\n"
+      "st1w { z29.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z30.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z31.s }, p0, [x21, #3, MUL VL]\n"
       "72:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 62b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 74f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 73f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "73:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "74:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp
index 3baf7b9715..71c6afba42 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -102,32 +102,32 @@ void sve_hybrid_fp32_mla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x9, %x[bias]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 3f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z8.b, #0x0\n"
@@ -135,175 +135,175 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
-      "cmp x26, #0x4\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
-      "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
+      "fmla z10.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n"
       "addvl x10, x10, #16\n"
-      "fmla z10.s, z6.s, z0.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
+      "add x26, x26, #0x10\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[0]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
-      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
+      "addvl x10, x10, #4\n"
       "ble 11f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
+      "addvl x10, x10, #4\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "13:"  // Height 1: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 16f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 16f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "addvl x12, x12, #4\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "b 18f\n"
       "17:"  // Height 2: no accumulate
       "mov z8.b, #0x0\n"
@@ -315,52 +315,52 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "19:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "21:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "cmp x26, #0x4\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
+      "cmp x27, #0x4\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n"
@@ -395,156 +395,156 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z15.s, z7.s, z1.s[3]\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "fmla z13.s, z7.s, z1.s[0]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "ble 24f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "ble 24f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x1\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "ble 24f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
-      "addvl x10, x10, #4\n"
       "fmla z10.s, z6.s, z0.s[3]\n"
       "fmla z14.s, z6.s, z1.s[3]\n"
+      "addvl x10, x10, #4\n"
       "fmla z11.s, z7.s, z0.s[3]\n"
       "fmla z15.s, z7.s, z1.s[3]\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 29f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 29f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
       "mov z13.d, z9.d\n"
-      "addvl x9, x9, #4\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
       "b 31f\n"
       "30:"  // Height 3: no accumulate
       "mov z8.b, #0x0\n"
@@ -560,63 +560,63 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "34:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "cmp x26, #0x4\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
+      "cmp x27, #0x4\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z14.s, z6.s, z1.s[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z18.s, z6.s, z2.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n"
@@ -624,32 +624,32 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z10.s, z6.s, z0.s[1]\n"
       "fmla z14.s, z6.s, z1.s[1]\n"
       "fmla z18.s, z6.s, z2.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[2]\n"
       "fmla z14.s, z6.s, z1.s[2]\n"
       "fmla z18.s, z6.s, z2.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n"
       "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n"
@@ -661,19 +661,19 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[3]\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
       "addvl x10, x10, #4\n"
@@ -685,13 +685,13 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[0]\n"
       "ble 37f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
@@ -704,13 +704,13 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[1]\n"
       "ble 37f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
@@ -723,12 +723,12 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[2]\n"
       "ble 37f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
@@ -740,116 +740,116 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z15.s, z7.s, z1.s[3]\n"
       "fmla z19.s, z7.s, z2.s[3]\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 42f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 42f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "b 44f\n"
       "43:"  // Height 4: no accumulate
       "mov z8.b, #0x0\n"
@@ -869,55 +869,55 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "47:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "cmp x26, #0x4\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
-      "add x22, x22, #0x10\n"
-      "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
@@ -991,21 +991,21 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[3]\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1020,13 +1020,13 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[0]\n"
       "ble 50f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
@@ -1043,13 +1043,13 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[1]\n"
       "ble 50f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
@@ -1066,8 +1066,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[2]\n"
       "ble 50f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
@@ -1087,103 +1087,103 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z19.s, z7.s, z2.s[3]\n"
       "fmla z23.s, z7.s, z3.s[3]\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 55f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 55f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -1194,31 +1194,31 @@ void sve_hybrid_fp32_mla_6x4VL (
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
       "b 57f\n"
       "56:"  // Height 5: no accumulate
       "mov z8.b, #0x0\n"
@@ -1242,61 +1242,61 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "60:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "cmp x26, #0x4\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
-      "add x22, x22, #0x10\n"
-      "fmla z17.s, z7.s, z2.s[0]\n"
-      "add x21, x21, #0x10\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
+      "add x25, x25, #0x10\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
+      "fmla z17.s, z7.s, z2.s[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
       "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n"
@@ -1305,8 +1305,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z18.s, z6.s, z2.s[0]\n"
       "fmla z22.s, z6.s, z3.s[0]\n"
       "fmla z26.s, z6.s, z4.s[0]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[0]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[0]\n"
       "fmla z19.s, z7.s, z2.s[0]\n"
       "fmla z23.s, z7.s, z3.s[0]\n"
@@ -1317,8 +1317,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
@@ -1330,8 +1330,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z18.s, z6.s, z2.s[1]\n"
       "fmla z22.s, z6.s, z3.s[1]\n"
       "fmla z26.s, z6.s, z4.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[1]\n"
       "fmla z19.s, z7.s, z2.s[1]\n"
       "fmla z23.s, z7.s, z3.s[1]\n"
@@ -1342,8 +1342,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
@@ -1354,8 +1354,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z18.s, z6.s, z2.s[2]\n"
       "fmla z22.s, z6.s, z3.s[2]\n"
       "fmla z26.s, z6.s, z4.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n"
       "fmla z15.s, z7.s, z1.s[2]\n"
       "fmla z19.s, z7.s, z2.s[2]\n"
       "fmla z23.s, z7.s, z3.s[2]\n"
@@ -1366,8 +1366,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
@@ -1385,23 +1385,23 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[3]\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
-      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
+      "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
@@ -1419,15 +1419,15 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[0]\n"
       "ble 63f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[1]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[1]\n"
       "fmla z17.s, z7.s, z2.s[1]\n"
       "fmla z21.s, z7.s, z3.s[1]\n"
@@ -1446,15 +1446,15 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[1]\n"
       "ble 63f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[2]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[2]\n"
       "fmla z17.s, z7.s, z2.s[2]\n"
       "fmla z21.s, z7.s, z3.s[2]\n"
@@ -1473,14 +1473,14 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[2]\n"
       "ble 63f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
       "fmla z24.s, z6.s, z4.s[3]\n"
-      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z9.s, z7.s, z0.s[3]\n"
+      "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
       "fmla z13.s, z7.s, z1.s[3]\n"
       "fmla z17.s, z7.s, z2.s[3]\n"
       "fmla z21.s, z7.s, z3.s[3]\n"
@@ -1498,119 +1498,119 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z23.s, z7.s, z3.s[3]\n"
       "fmla z27.s, z7.s, z4.s[3]\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
       "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[bias]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x11\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x11\n"
-      "cbz x9, 68f\n"
-      "ld1w { z8.s }, p5/Z, [x9]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 68f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "mov z12.d, z8.d\n"
-      "ld1w { z9.s }, p5/Z, [x9, #1, MUL VL]\n"
-      "mov z16.d, z8.d\n"
-      "ld1w { z10.s }, p5/Z, [x9, #2, MUL VL]\n"
-      "mov z20.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
       "mov z13.d, z9.d\n"
-      "mov z17.d, z9.d\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "mov z14.d, z10.d\n"
       "mov z15.d, z11.d\n"
+      "mov z16.d, z8.d\n"
+      "mov z17.d, z9.d\n"
+      "addvl x12, x12, #4\n"
       "mov z18.d, z10.d\n"
       "mov z19.d, z11.d\n"
+      "mov z20.d, z8.d\n"
       "mov z21.d, z9.d\n"
       "mov z22.d, z10.d\n"
       "mov z23.d, z11.d\n"
@@ -1625,36 +1625,36 @@ void sve_hybrid_fp32_mla_6x4VL (
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x23]\n"
-      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x21]\n"
-      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x24]\n"
+      "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x22]\n"
+      "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "b 70f\n"
       "69:"  // Height 6: no accumulate
       "mov z8.b, #0x0\n"
@@ -1682,67 +1682,67 @@ void sve_hybrid_fp32_mla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "71:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "73:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "cmp x26, #0x4\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
+      "fmla z20.s, z6.s, z3.s[0]\n"
       "add x23, x23, #0x10\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      "fmla z20.s, z6.s, z3.s[0]\n"
-      "add x21, x21, #0x10\n"
-      "fmla z17.s, z7.s, z2.s[0]\n"
-      "add x20, x20, #0x10\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z28.s, z6.s, z5.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
+      "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
       "fmla z29.s, z7.s, z5.s[0]\n"
@@ -1846,25 +1846,25 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z31.s, z7.s, z5.s[3]\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "whilelt p0.s, XZR, x26\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
       "fmla z8.s, z6.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
-      "fmla z9.s, z7.s, z0.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
       "fmla z12.s, z6.s, z1.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "fmla z13.s, z7.s, z1.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
       "fmla z16.s, z6.s, z2.s[0]\n"
       "fmla z20.s, z6.s, z3.s[0]\n"
       "fmla z24.s, z6.s, z4.s[0]\n"
       "fmla z28.s, z6.s, z5.s[0]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
+      "fmla z9.s, z7.s, z0.s[0]\n"
+      "fmla z13.s, z7.s, z1.s[0]\n"
       "fmla z17.s, z7.s, z2.s[0]\n"
       "fmla z21.s, z7.s, z3.s[0]\n"
       "fmla z25.s, z7.s, z4.s[0]\n"
@@ -1885,12 +1885,12 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z31.s, z7.s, z5.s[0]\n"
       "ble 76f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[1]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[1]\n"
       "fmla z12.s, z6.s, z1.s[1]\n"
       "fmla z16.s, z6.s, z2.s[1]\n"
       "fmla z20.s, z6.s, z3.s[1]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, z6.s, z4.s[1]\n"
       "fmla z28.s, z6.s, z5.s[1]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
@@ -1916,12 +1916,12 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z31.s, z7.s, z5.s[1]\n"
       "ble 76f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[2]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
-      "subs x26, x26, #0x1\n"
+      "fmla z8.s, z6.s, z0.s[2]\n"
       "fmla z12.s, z6.s, z1.s[2]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
       "fmla z20.s, z6.s, z3.s[2]\n"
+      "subs x27, x27, #0x1\n"
       "fmla z24.s, z6.s, z4.s[2]\n"
       "fmla z28.s, z6.s, z5.s[2]\n"
       "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n"
@@ -1947,8 +1947,8 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z31.s, z7.s, z5.s[2]\n"
       "ble 76f\n"
       "ld1w { z6.s }, p5/Z, [x10]\n"
-      "fmla z8.s, z6.s, z0.s[3]\n"
       "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n"
+      "fmla z8.s, z6.s, z0.s[3]\n"
       "fmla z12.s, z6.s, z1.s[3]\n"
       "fmla z16.s, z6.s, z2.s[3]\n"
       "fmla z20.s, z6.s, z3.s[3]\n"
@@ -1976,115 +1976,115 @@ void sve_hybrid_fp32_mla_6x4VL (
       "fmla z27.s, z7.s, z4.s[3]\n"
       "fmla z31.s, z7.s, z5.s[3]\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 71b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
-      "fmin z8.s, p5/M, z8.s, z0.s\n"
-      "fmin z9.s, p5/M, z9.s, z0.s\n"
-      "fmin z10.s, p5/M, z10.s, z0.s\n"
-      "fmin z11.s, p5/M, z11.s, z0.s\n"
-      "fmin z12.s, p5/M, z12.s, z0.s\n"
-      "fmax z8.s, p5/M, z8.s, z1.s\n"
-      "fmax z9.s, p5/M, z9.s, z1.s\n"
-      "fmax z10.s, p5/M, z10.s, z1.s\n"
-      "fmax z11.s, p5/M, z11.s, z1.s\n"
-      "fmax z12.s, p5/M, z12.s, z1.s\n"
-      "fmin z13.s, p5/M, z13.s, z0.s\n"
-      "fmin z14.s, p5/M, z14.s, z0.s\n"
-      "fmin z15.s, p5/M, z15.s, z0.s\n"
-      "fmin z16.s, p5/M, z16.s, z0.s\n"
-      "fmax z13.s, p5/M, z13.s, z1.s\n"
-      "fmax z14.s, p5/M, z14.s, z1.s\n"
-      "fmax z15.s, p5/M, z15.s, z1.s\n"
-      "fmax z16.s, p5/M, z16.s, z1.s\n"
-      "fmin z17.s, p5/M, z17.s, z0.s\n"
-      "fmin z18.s, p5/M, z18.s, z0.s\n"
-      "fmin z19.s, p5/M, z19.s, z0.s\n"
-      "fmin z20.s, p5/M, z20.s, z0.s\n"
-      "fmax z17.s, p5/M, z17.s, z1.s\n"
-      "fmax z18.s, p5/M, z18.s, z1.s\n"
-      "fmax z19.s, p5/M, z19.s, z1.s\n"
-      "fmax z20.s, p5/M, z20.s, z1.s\n"
-      "fmin z21.s, p5/M, z21.s, z0.s\n"
-      "fmin z22.s, p5/M, z22.s, z0.s\n"
-      "fmin z23.s, p5/M, z23.s, z0.s\n"
-      "fmin z24.s, p5/M, z24.s, z0.s\n"
-      "fmax z21.s, p5/M, z21.s, z1.s\n"
-      "fmax z22.s, p5/M, z22.s, z1.s\n"
-      "fmax z23.s, p5/M, z23.s, z1.s\n"
-      "fmax z24.s, p5/M, z24.s, z1.s\n"
-      "fmin z25.s, p5/M, z25.s, z0.s\n"
-      "fmin z26.s, p5/M, z26.s, z0.s\n"
-      "fmin z27.s, p5/M, z27.s, z0.s\n"
-      "fmin z28.s, p5/M, z28.s, z0.s\n"
-      "fmax z25.s, p5/M, z25.s, z1.s\n"
-      "fmax z26.s, p5/M, z26.s, z1.s\n"
-      "fmax z27.s, p5/M, z27.s, z1.s\n"
-      "fmax z28.s, p5/M, z28.s, z1.s\n"
-      "fmin z29.s, p5/M, z29.s, z0.s\n"
-      "fmin z30.s, p5/M, z30.s, z0.s\n"
-      "fmin z31.s, p5/M, z31.s, z0.s\n"
-      "fmax z29.s, p5/M, z29.s, z1.s\n"
-      "fmax z30.s, p5/M, z30.s, z1.s\n"
-      "fmax z31.s, p5/M, z31.s, z1.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
+      "fmin z8.s, p5/M, z8.s, z1.s\n"
+      "fmin z9.s, p5/M, z9.s, z1.s\n"
+      "fmin z10.s, p5/M, z10.s, z1.s\n"
+      "fmin z11.s, p5/M, z11.s, z1.s\n"
+      "fmin z12.s, p5/M, z12.s, z1.s\n"
+      "fmin z13.s, p5/M, z13.s, z1.s\n"
+      "fmin z14.s, p5/M, z14.s, z1.s\n"
+      "fmin z15.s, p5/M, z15.s, z1.s\n"
+      "fmin z16.s, p5/M, z16.s, z1.s\n"
+      "fmin z17.s, p5/M, z17.s, z1.s\n"
+      "fmin z18.s, p5/M, z18.s, z1.s\n"
+      "fmin z19.s, p5/M, z19.s, z1.s\n"
+      "fmin z20.s, p5/M, z20.s, z1.s\n"
+      "fmin z21.s, p5/M, z21.s, z1.s\n"
+      "fmin z22.s, p5/M, z22.s, z1.s\n"
+      "fmin z23.s, p5/M, z23.s, z1.s\n"
+      "fmin z24.s, p5/M, z24.s, z1.s\n"
+      "fmin z25.s, p5/M, z25.s, z1.s\n"
+      "fmin z26.s, p5/M, z26.s, z1.s\n"
+      "fmin z27.s, p5/M, z27.s, z1.s\n"
+      "fmin z28.s, p5/M, z28.s, z1.s\n"
+      "fmin z29.s, p5/M, z29.s, z1.s\n"
+      "fmin z30.s, p5/M, z30.s, z1.s\n"
+      "fmin z31.s, p5/M, z31.s, z1.s\n"
+      "fmax z8.s, p5/M, z8.s, z0.s\n"
+      "fmax z9.s, p5/M, z9.s, z0.s\n"
+      "fmax z10.s, p5/M, z10.s, z0.s\n"
+      "fmax z11.s, p5/M, z11.s, z0.s\n"
+      "fmax z12.s, p5/M, z12.s, z0.s\n"
+      "fmax z13.s, p5/M, z13.s, z0.s\n"
+      "fmax z14.s, p5/M, z14.s, z0.s\n"
+      "fmax z15.s, p5/M, z15.s, z0.s\n"
+      "fmax z16.s, p5/M, z16.s, z0.s\n"
+      "fmax z17.s, p5/M, z17.s, z0.s\n"
+      "fmax z18.s, p5/M, z18.s, z0.s\n"
+      "fmax z19.s, p5/M, z19.s, z0.s\n"
+      "fmax z20.s, p5/M, z20.s, z0.s\n"
+      "fmax z21.s, p5/M, z21.s, z0.s\n"
+      "fmax z22.s, p5/M, z22.s, z0.s\n"
+      "fmax z23.s, p5/M, z23.s, z0.s\n"
+      "fmax z24.s, p5/M, z24.s, z0.s\n"
+      "fmax z25.s, p5/M, z25.s, z0.s\n"
+      "fmax z26.s, p5/M, z26.s, z0.s\n"
+      "fmax z27.s, p5/M, z27.s, z0.s\n"
+      "fmax z28.s, p5/M, z28.s, z0.s\n"
+      "fmax z29.s, p5/M, z29.s, z0.s\n"
+      "fmax z30.s, p5/M, z30.s, z0.s\n"
+      "fmax z31.s, p5/M, z31.s, z0.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x24]\n"
-      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x22]\n"
-      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z28.s }, p4, [x20]\n"
-      "st1w { z29.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z30.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z31.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x25]\n"
+      "st1w { z13.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x23]\n"
+      "st1w { z21.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z28.s }, p4, [x21]\n"
+      "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
       "decw x11, ALL, MUL #4\n"
       "cmp x11, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp
index 0a37f8abfc..2ccd050f18 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,248 +105,248 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 25f\n"
       "beq 13f\n"
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 3f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
-      "addvl x13, x13, #1\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 3f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
+      "addvl x14, x14, #1\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z24.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "6:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "cbnz x9, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "cbnz x10, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x27, %x[input_ptr]\n"
+      "mov x28, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 11f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmax z24.s, p1/M, z24.s, z16.s\n"
       "11:"  // Height 1: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
       "12:"  // Height 1: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 2b\n"
       "b 98f\n"
       "13:"  // Height 2
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "14:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 15f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 15f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "b 17f\n"
       "15:"  // Height 2: no bias
       "tbz %x[flags], #0, 16f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
       "b 17f\n"
       "16:"  // Height 2: no accumulate
       "mov z24.b, #0x0\n"
       "mov z25.b, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "18:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "cbnz x9, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "cbnz x10, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
       "20:"  // Height 2: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
       "ble 22f\n"
       "21:"  // Height 2: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
-      "add x26, x26, #0x4\n"
+      "add x27, x27, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
       "bgt 21b\n"
       "22:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "bne 18b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
       "tbz %x[flags], #1, 23f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmax z24.s, p1/M, z24.s, z16.s\n"
       "fmax z25.s, p1/M, z25.s, z16.s\n"
       "23:"  // Height 2: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
       "24:"  // Height 2: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 14b\n"
       "b 98f\n"
       "25:"  // Height 3
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "26:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 27f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 27f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "b 29f\n"
       "27:"  // Height 3: no bias
       "tbz %x[flags], #0, 28f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
       "b 29f\n"
       "28:"  // Height 3: no accumulate
       "mov z24.b, #0x0\n"
       "mov z25.b, #0x0\n"
       "mov z26.b, #0x0\n"
       "29:"  // Height 3: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "30:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "cbnz x9, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "cbnz x10, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 32f\n"
       "31:"  // Height 3: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "32:"  // Height 3: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
       "ble 34f\n"
       "33:"  // Height 3: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
       "bgt 33b\n"
       "34:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "tbz %x[flags], #1, 35f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -354,40 +354,40 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z25.s, p1/M, z25.s, z16.s\n"
       "fmax z26.s, p1/M, z26.s, z16.s\n"
       "35:"  // Height 3: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
       "36:"  // Height 3: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 26b\n"
       "b 98f\n"
       "37:"  // Height 4
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "38:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 39f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 39f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "b 41f\n"
       "39:"  // Height 4: no bias
       "tbz %x[flags], #0, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
-      "ld1w { z27.s }, p0/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
+      "ld1w { z27.s }, p0/Z, [x25]\n"
       "b 41f\n"
       "40:"  // Height 4: no accumulate
       "mov z24.b, #0x0\n"
@@ -395,74 +395,74 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "41:"  // Height 4: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "42:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 43f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x9, 44f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x10, 44f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 44f\n"
       "43:"  // Height 4: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "44:"  // Height 4: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
       "ble 46f\n"
       "45:"  // Height 4: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
-      "add x24, x24, #0x4\n"
+      "add x25, x25, #0x4\n"
       "fmla z27.s, p1/M, z8.s, z3.s\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
       "bgt 45b\n"
       "46:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "fmla z27.s, p1/M, z9.s, z3.s\n"
       "bne 42b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "tbz %x[flags], #1, 47f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -472,44 +472,44 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z26.s, p1/M, z26.s, z16.s\n"
       "fmax z27.s, p1/M, z27.s, z16.s\n"
       "47:"  // Height 4: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
-      "st1w { z27.s }, p0, [x24]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
+      "st1w { z27.s }, p0, [x25]\n"
       "48:"  // Height 4: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 38b\n"
       "b 98f\n"
       "49:"  // Height 5
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "50:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 51f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 51f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "b 53f\n"
       "51:"  // Height 5: no bias
       "tbz %x[flags], #0, 52f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
-      "ld1w { z27.s }, p0/Z, [x24]\n"
-      "ld1w { z28.s }, p0/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
+      "ld1w { z27.s }, p0/Z, [x25]\n"
+      "ld1w { z28.s }, p0/Z, [x24]\n"
       "b 53f\n"
       "52:"  // Height 5: no accumulate
       "mov z24.b, #0x0\n"
@@ -518,83 +518,83 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "mov z27.b, #0x0\n"
       "mov z28.b, #0x0\n"
       "53:"  // Height 5: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "54:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 55f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "cbnz x9, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "cbnz x10, 56f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 56f\n"
       "55:"  // Height 5: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "56:"  // Height 5: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
       "ble 58f\n"
       "57:"  // Height 5: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z27.s, p1/M, z8.s, z3.s\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "addvl x11, x11, #1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, p1/M, z8.s, z4.s\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
       "bgt 57b\n"
       "58:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "fmla z27.s, p1/M, z9.s, z3.s\n"
       "fmla z28.s, p1/M, z9.s, z4.s\n"
       "bne 54b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 59f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -606,48 +606,48 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z27.s, p1/M, z27.s, z16.s\n"
       "fmax z28.s, p1/M, z28.s, z16.s\n"
       "59:"  // Height 5: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
-      "st1w { z27.s }, p0, [x24]\n"
-      "st1w { z28.s }, p0, [x23]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
+      "st1w { z27.s }, p0, [x25]\n"
+      "st1w { z28.s }, p0, [x24]\n"
       "60:"  // Height 5: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 50b\n"
       "b 98f\n"
       "61:"  // Height 6
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "62:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 63f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 63f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
       "b 65f\n"
       "63:"  // Height 6: no bias
       "tbz %x[flags], #0, 64f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
-      "ld1w { z27.s }, p0/Z, [x24]\n"
-      "ld1w { z28.s }, p0/Z, [x23]\n"
-      "ld1w { z29.s }, p0/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z27.s }, p0/Z, [x25]\n"
+      "ld1w { z28.s }, p0/Z, [x24]\n"
+      "ld1w { z29.s }, p0/Z, [x23]\n"
       "b 65f\n"
       "64:"  // Height 6: no accumulate
       "mov z24.b, #0x0\n"
@@ -657,92 +657,92 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "mov z28.b, #0x0\n"
       "mov z29.b, #0x0\n"
       "65:"  // Height 6: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "66:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "cbnz x9, 68f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "cbnz x10, 68f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 68f\n"
       "67:"  // Height 6: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "68:"  // Height 6: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
       "ble 70f\n"
       "69:"  // Height 6: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z27.s, p1/M, z8.s, z3.s\n"
       "fmla z28.s, p1/M, z8.s, z4.s\n"
-      "add x22, x22, #0x4\n"
-      "addvl x11, x11, #1\n"
+      "add x23, x23, #0x4\n"
+      "addvl x12, x12, #1\n"
       "fmla z29.s, p1/M, z8.s, z5.s\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
       "bgt 69b\n"
       "70:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "fmla z27.s, p1/M, z9.s, z3.s\n"
       "fmla z28.s, p1/M, z9.s, z4.s\n"
       "fmla z29.s, p1/M, z9.s, z5.s\n"
       "bne 66b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 71f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -756,31 +756,31 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z28.s, p1/M, z28.s, z16.s\n"
       "fmax z29.s, p1/M, z29.s, z16.s\n"
       "71:"  // Height 6: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
-      "st1w { z27.s }, p0, [x24]\n"
-      "st1w { z28.s }, p0, [x23]\n"
-      "st1w { z29.s }, p0, [x22]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
+      "st1w { z27.s }, p0, [x25]\n"
+      "st1w { z28.s }, p0, [x24]\n"
+      "st1w { z29.s }, p0, [x23]\n"
       "72:"  // Height 6: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 62b\n"
       "b 98f\n"
       "73:"  // Height 7
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
       "74:"  // Height 7: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 75f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 75f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
@@ -788,20 +788,20 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "b 77f\n"
       "75:"  // Height 7: no bias
       "tbz %x[flags], #0, 76f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
-      "ld1w { z27.s }, p0/Z, [x24]\n"
-      "ld1w { z28.s }, p0/Z, [x23]\n"
-      "ld1w { z29.s }, p0/Z, [x22]\n"
-      "ld1w { z30.s }, p0/Z, [x21]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z27.s }, p0/Z, [x25]\n"
+      "ld1w { z28.s }, p0/Z, [x24]\n"
+      "ld1w { z29.s }, p0/Z, [x23]\n"
+      "ld1w { z30.s }, p0/Z, [x22]\n"
       "b 77f\n"
       "76:"  // Height 7: no accumulate
       "mov z24.b, #0x0\n"
@@ -812,101 +812,101 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "mov z29.b, #0x0\n"
       "mov z30.b, #0x0\n"
       "77:"  // Height 7: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "78:"  // Height 7: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "cbnz x9, 80f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "cbnz x10, 80f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 80f\n"
       "79:"  // Height 7: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "80:"  // Height 7: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
-      "ld1rw { z6.s }, p1/Z, [x21]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
+      "ld1rw { z6.s }, p1/Z, [x22]\n"
       "ble 82f\n"
       "81:"  // Height 7: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z27.s, p1/M, z8.s, z3.s\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "add x23, x23, #0x4\n"
       "add x22, x22, #0x4\n"
-      "add x21, x21, #0x4\n"
       "fmla z28.s, p1/M, z8.s, z4.s\n"
       "fmla z29.s, p1/M, z8.s, z5.s\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
       "fmla z30.s, p1/M, z8.s, z6.s\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
-      "ld1rw { z6.s }, p1/Z, [x21]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
+      "ld1rw { z6.s }, p1/Z, [x22]\n"
       "bgt 81b\n"
       "82:"  // Height 7: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "fmla z27.s, p1/M, z9.s, z3.s\n"
       "fmla z28.s, p1/M, z9.s, z4.s\n"
       "fmla z29.s, p1/M, z9.s, z5.s\n"
       "fmla z30.s, p1/M, z9.s, z6.s\n"
       "bne 78b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 83f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -922,35 +922,35 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z29.s, p1/M, z29.s, z16.s\n"
       "fmax z30.s, p1/M, z30.s, z16.s\n"
       "83:"  // Height 7: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
-      "st1w { z27.s }, p0, [x24]\n"
-      "st1w { z28.s }, p0, [x23]\n"
-      "st1w { z29.s }, p0, [x22]\n"
-      "st1w { z30.s }, p0, [x21]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
+      "st1w { z27.s }, p0, [x25]\n"
+      "st1w { z28.s }, p0, [x24]\n"
+      "st1w { z29.s }, p0, [x23]\n"
+      "st1w { z30.s }, p0, [x22]\n"
       "84:"  // Height 7: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 74b\n"
       "b 98f\n"
       "85:"  // Height 8
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x20\n"
-      "mov x13, %x[bias]\n"
-      "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x20\n"
+      "mov x14, %x[bias]\n"
+      "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "86:"  // Height 8: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p0.s, x19, x12\n"
-      "cbz x13, 87f\n"
-      "ld1w { z24.s }, p1/Z, [x13]\n"
+      "mov x20, #0x0\n"
+      "whilelt p0.s, x20, x13\n"
+      "cbz x14, 87f\n"
+      "ld1w { z24.s }, p1/Z, [x14]\n"
       "mov z25.d, z24.d\n"
       "mov z26.d, z24.d\n"
-      "addvl x13, x13, #1\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
@@ -959,22 +959,22 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "b 89f\n"
       "87:"  // Height 8: no bias
       "tbz %x[flags], #0, 88f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z24.s }, p0/Z, [x10]\n"
-      "ld1w { z25.s }, p0/Z, [x26]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z26.s }, p0/Z, [x25]\n"
-      "ld1w { z27.s }, p0/Z, [x24]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z28.s }, p0/Z, [x23]\n"
-      "ld1w { z29.s }, p0/Z, [x22]\n"
-      "ld1w { z30.s }, p0/Z, [x21]\n"
-      "ld1w { z31.s }, p0/Z, [x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p0/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p0/Z, [x27]\n"
+      "ld1w { z26.s }, p0/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z27.s }, p0/Z, [x25]\n"
+      "ld1w { z28.s }, p0/Z, [x24]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z29.s }, p0/Z, [x23]\n"
+      "ld1w { z30.s }, p0/Z, [x22]\n"
+      "ld1w { z31.s }, p0/Z, [x21]\n"
       "b 89f\n"
       "88:"  // Height 8: no accumulate
       "mov z24.b, #0x0\n"
@@ -986,90 +986,90 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "89:"  // Height 8: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "90:"  // Height 8: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w28, [x19, x9, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 91f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "ldr x20, [x20, #0x38]\n"
-      "cbnz x9, 92f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "ldr x21, [x21, #0x38]\n"
+      "cbnz x10, 92f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 92f\n"
       "91:"  // Height 8: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "92:"  // Height 8: input setup done
-      "subs x28, x28, #0x1\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
-      "ld1rw { z6.s }, p1/Z, [x21]\n"
-      "ld1rw { z7.s }, p1/Z, [x20]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
+      "ld1rw { z6.s }, p1/Z, [x22]\n"
+      "ld1rw { z7.s }, p1/Z, [x21]\n"
       "ble 94f\n"
       "93:"  // Height 8: Multiply loop: Main loop
-      "ld1w { z8.s }, p1/Z, [x11]\n"
-      "add x27, x27, #0x4\n"
-      "subs x28, x28, #0x1\n"
+      "ld1w { z8.s }, p1/Z, [x12]\n"
+      "add x28, x28, #0x4\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, p1/M, z8.s, z0.s\n"
+      "add x27, x27, #0x4\n"
       "add x26, x26, #0x4\n"
-      "add x25, x25, #0x4\n"
       "fmla z25.s, p1/M, z8.s, z1.s\n"
       "fmla z26.s, p1/M, z8.s, z2.s\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "fmla z27.s, p1/M, z8.s, z3.s\n"
       "fmla z28.s, p1/M, z8.s, z4.s\n"
+      "add x23, x23, #0x4\n"
       "add x22, x22, #0x4\n"
-      "add x21, x21, #0x4\n"
       "fmla z29.s, p1/M, z8.s, z5.s\n"
-      "ld1rw { z0.s }, p1/Z, [x27]\n"
-      "add x20, x20, #0x4\n"
-      "addvl x11, x11, #1\n"
-      "ld1rw { z1.s }, p1/Z, [x26]\n"
+      "ld1rw { z0.s }, p1/Z, [x28]\n"
+      "add x21, x21, #0x4\n"
+      "addvl x12, x12, #1\n"
+      "ld1rw { z1.s }, p1/Z, [x27]\n"
       "fmla z30.s, p1/M, z8.s, z6.s\n"
       "fmla z31.s, p1/M, z8.s, z7.s\n"
-      "ld1rw { z2.s }, p1/Z, [x25]\n"
-      "ld1rw { z3.s }, p1/Z, [x24]\n"
-      "ld1rw { z4.s }, p1/Z, [x23]\n"
-      "ld1rw { z5.s }, p1/Z, [x22]\n"
-      "ld1rw { z6.s }, p1/Z, [x21]\n"
-      "ld1rw { z7.s }, p1/Z, [x20]\n"
+      "ld1rw { z2.s }, p1/Z, [x26]\n"
+      "ld1rw { z3.s }, p1/Z, [x25]\n"
+      "ld1rw { z4.s }, p1/Z, [x24]\n"
+      "ld1rw { z5.s }, p1/Z, [x23]\n"
+      "ld1rw { z6.s }, p1/Z, [x22]\n"
+      "ld1rw { z7.s }, p1/Z, [x21]\n"
       "bgt 93b\n"
       "94:"  // Height 8: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "ld1w { z9.s }, p1/Z, [x11]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ld1w { z9.s }, p1/Z, [x12]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "fmla z24.s, p1/M, z9.s, z0.s\n"
       "fmla z25.s, p1/M, z9.s, z1.s\n"
-      "addvl x11, x11, #1\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, p1/M, z9.s, z2.s\n"
       "fmla z27.s, p1/M, z9.s, z3.s\n"
       "fmla z28.s, p1/M, z9.s, z4.s\n"
@@ -1077,19 +1077,19 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmla z30.s, p1/M, z9.s, z6.s\n"
       "fmla z31.s, p1/M, z9.s, z7.s\n"
       "bne 90b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 95f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z17.s }, p1/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z16.s }, p1/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p1/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p1/Z, [x20]\n"
       "fmin z24.s, p1/M, z24.s, z17.s\n"
       "fmin z25.s, p1/M, z25.s, z17.s\n"
       "fmin z26.s, p1/M, z26.s, z17.s\n"
@@ -1107,35 +1107,35 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx (
       "fmax z30.s, p1/M, z30.s, z16.s\n"
       "fmax z31.s, p1/M, z31.s, z16.s\n"
       "95:"  // Height 8: No activation
-      "st1w { z24.s }, p0, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p0, [x26]\n"
-      "st1w { z26.s }, p0, [x25]\n"
-      "st1w { z27.s }, p0, [x24]\n"
-      "st1w { z28.s }, p0, [x23]\n"
-      "st1w { z29.s }, p0, [x22]\n"
-      "st1w { z30.s }, p0, [x21]\n"
-      "st1w { z31.s }, p0, [x20]\n"
+      "st1w { z24.s }, p0, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p0, [x27]\n"
+      "st1w { z26.s }, p0, [x26]\n"
+      "st1w { z27.s }, p0, [x25]\n"
+      "st1w { z28.s }, p0, [x24]\n"
+      "st1w { z29.s }, p0, [x23]\n"
+      "st1w { z30.s }, p0, [x22]\n"
+      "st1w { z31.s }, p0, [x21]\n"
       "96:"  // Height 8: Writeback done
-      "decw x12\n"
-      "cmp x12, XZR\n"
+      "decw x13\n"
+      "cmp x13, XZR\n"
       "bgt 86b\n"
       "subs %x[M], %x[M], #0x8\n"
       "beq 98f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 97f\n"
-      "add x20, x20, #0x8\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x8\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "97:"  // Update direct input
-      "mov x19, #0x20\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x20\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "98:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp
index 5b4b6b9b2e..9679d49506 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -105,382 +105,382 @@ void sve_hybrid_fp32_mla_8x1VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x11, %x[bias]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 3f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
-      "addvl x11, x11, #1\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 3f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
+      "addvl x14, x14, #1\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
       "b 5f\n"
       "4:"  // Height 1: no accumulate
       "mov z24.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "6:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "cbnz x9, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "cbnz x10, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x27, %x[input_ptr]\n"
+      "mov x28, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "cmp x28, #0x4\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x27, x27, #0x10\n"
+      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "addvl x12, x12, #4\n"
+      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
+      "sub x9, x9, #0x4\n"
+      "cmp x9, #0x4\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
+      "add x28, x28, #0x10\n"
+      "addvl x12, x12, #4\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
       "addvl x12, x12, #1\n"
       "ble 11f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "addvl x12, x12, #1\n"
       "ble 11f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "addvl x12, x12, #1\n"
       "ble 11f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
       "addvl x12, x12, #1\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 6b\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
       "13:"  // Height 1: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 2b\n"
       "b 106f\n"
       "14:"  // Height 2
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 16f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 16f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
+      "addvl x14, x14, #1\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
       "b 18f\n"
       "17:"  // Height 2: no accumulate
       "mov z24.b, #0x0\n"
       "mov z25.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "19:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "cbnz x9, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "cbnz x10, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
       "21:"  // Height 2: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x27, x27, #0x10\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x26, x26, #0x10\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #4\n"
+      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "add x27, x27, #0x10\n"
+      "addvl x12, x12, #4\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
+      "addvl x12, x12, #1\n"
       "ble 24f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
+      "addvl x12, x12, #1\n"
       "ble 24f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
       "addvl x12, x12, #1\n"
       "ble 24f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
       "26:"  // Height 2: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 15b\n"
       "b 106f\n"
       "27:"  // Height 3
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 29f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 29f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
       "b 31f\n"
       "30:"  // Height 3: no accumulate
       "mov z24.b, #0x0\n"
       "mov z25.b, #0x0\n"
       "mov z26.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "cbnz x9, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "cbnz x10, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "34:"  // Height 3: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
+      "fmla z26.s, z8.s, z2.s[0]\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
       "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x26, x26, #0x10\n"
-      "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #4\n"
-      "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
+      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
+      "cmp x9, #0x4\n"
+      "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "fmla z24.s, z11.s, z0.s[3]\n"
+      "add x28, x28, #0x10\n"
+      "add x27, x27, #0x10\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
+      "fmla z24.s, z11.s, z0.s[3]\n"
+      "add x26, x26, #0x10\n"
+      "addvl x12, x12, #4\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
+      "addvl x12, x12, #1\n"
       "ble 37f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
+      "addvl x12, x12, #1\n"
       "ble 37f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
+      "addvl x12, x12, #1\n"
       "ble 37f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
       "39:"  // Height 3: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 28b\n"
       "b 106f\n"
       "40:"  // Height 4
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 42f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 42f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z27.s }, p1/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
+      "ld1w { z27.s }, p1/Z, [x25]\n"
       "b 44f\n"
       "43:"  // Height 4: no accumulate
       "mov z24.b, #0x0\n"
@@ -488,164 +488,164 @@ void sve_hybrid_fp32_mla_8x1VL (
       "mov z26.b, #0x0\n"
       "mov z27.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "45:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "cbnz x9, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "cbnz x10, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "47:"  // Height 4: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
-      "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
-      "add x26, x26, #0x10\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x25, x25, #0x10\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
+      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x24, x24, #0x10\n"
+      "fmla z24.s, z9.s, z0.s[1]\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #4\n"
-      "fmla z24.s, z10.s, z0.s[2]\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "add x25, x25, #0x10\n"
+      "addvl x12, x12, #4\n"
+      "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "fmla z24.s, z11.s, z0.s[3]\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "fmla z24.s, z11.s, z0.s[3]\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
       "ble 50f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "addvl x12, x12, #1\n"
       "ble 50f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "addvl x12, x12, #1\n"
       "ble 50f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmin z27.s, p2/M, z27.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
-      "fmax z27.s, p2/M, z27.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmin z27.s, p2/M, z27.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
+      "fmax z27.s, p2/M, z27.s, z16.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
-      "st1w { z27.s }, p1, [x24]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
+      "st1w { z27.s }, p1, [x25]\n"
       "52:"  // Height 4: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 41b\n"
       "b 106f\n"
       "53:"  // Height 5
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 55f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 55f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z27.s }, p1/Z, [x24]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z28.s }, p1/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
+      "ld1w { z27.s }, p1/Z, [x25]\n"
+      "ld1w { z28.s }, p1/Z, [x24]\n"
       "b 57f\n"
       "56:"  // Height 5: no accumulate
       "mov z24.b, #0x0\n"
@@ -654,185 +654,185 @@ void sve_hybrid_fp32_mla_8x1VL (
       "mov z27.b, #0x0\n"
       "mov z28.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "58:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "cbnz x9, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "cbnz x10, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "60:"  // Height 5: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
-      "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
-      "add x26, x26, #0x10\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z25.s, z9.s, z1.s[1]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x23, x23, #0x10\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
-      "addvl x12, x12, #4\n"
+      "fmla z24.s, z9.s, z0.s[1]\n"
+      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
+      "add x27, x27, #0x10\n"
+      "fmla z25.s, z9.s, z1.s[1]\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
-      "fmla z24.s, z10.s, z0.s[2]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
-      "fmla z25.s, z10.s, z1.s[2]\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
+      "add x24, x24, #0x10\n"
+      "addvl x12, x12, #4\n"
+      "fmla z24.s, z10.s, z0.s[2]\n"
+      "fmla z25.s, z10.s, z1.s[2]\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
-      "fmla z24.s, z11.s, z0.s[3]\n"
       "fmla z28.s, z10.s, z4.s[2]\n"
+      "fmla z24.s, z11.s, z0.s[3]\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "fmla z28.s, z11.s, z4.s[3]\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
       "ble 63f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "ble 63f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z10.s, z4.s[2]\n"
       "ble 63f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "fmla z28.s, z11.s, z4.s[3]\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmin z27.s, p2/M, z27.s, z16.s\n"
-      "fmin z28.s, p2/M, z28.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
-      "fmax z27.s, p2/M, z27.s, z17.s\n"
-      "fmax z28.s, p2/M, z28.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmin z27.s, p2/M, z27.s, z17.s\n"
+      "fmin z28.s, p2/M, z28.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
+      "fmax z27.s, p2/M, z27.s, z16.s\n"
+      "fmax z28.s, p2/M, z28.s, z16.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
-      "st1w { z27.s }, p1, [x24]\n"
-      "st1w { z28.s }, p1, [x23]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
+      "st1w { z27.s }, p1, [x25]\n"
+      "st1w { z28.s }, p1, [x24]\n"
       "65:"  // Height 5: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 54b\n"
       "b 106f\n"
       "66:"  // Height 6
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 68f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 68f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z27.s }, p1/Z, [x24]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z28.s }, p1/Z, [x23]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z29.s }, p1/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z27.s }, p1/Z, [x25]\n"
+      "ld1w { z28.s }, p1/Z, [x24]\n"
+      "ld1w { z29.s }, p1/Z, [x23]\n"
       "b 70f\n"
       "69:"  // Height 6: no accumulate
       "mov z24.b, #0x0\n"
@@ -842,73 +842,73 @@ void sve_hybrid_fp32_mla_8x1VL (
       "mov z28.b, #0x0\n"
       "mov z29.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "71:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "cbnz x9, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "cbnz x10, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "73:"  // Height 6: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
-      "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
-      "add x26, x26, #0x10\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z25.s, z9.s, z1.s[1]\n"
       "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z28.s, z8.s, z4.s[0]\n"
       "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x22, x22, #0x10\n"
+      "fmla z28.s, z8.s, z4.s[0]\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "fmla z24.s, z9.s, z0.s[1]\n"
+      "fmla z25.s, z9.s, z1.s[1]\n"
+      "add x23, x23, #0x10\n"
       "addvl x12, x12, #4\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
-      "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "fmla z29.s, z9.s, z5.s[1]\n"
+      "fmla z24.s, z10.s, z0.s[2]\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
@@ -922,105 +922,105 @@ void sve_hybrid_fp32_mla_8x1VL (
       "fmla z29.s, z11.s, z5.s[3]\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
       "ble 76f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "fmla z29.s, z9.s, z5.s[1]\n"
       "ble 76f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z10.s, z4.s[2]\n"
       "fmla z29.s, z10.s, z5.s[2]\n"
       "ble 76f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "fmla z28.s, z11.s, z4.s[3]\n"
       "fmla z29.s, z11.s, z5.s[3]\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 71b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmin z27.s, p2/M, z27.s, z16.s\n"
-      "fmin z28.s, p2/M, z28.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
-      "fmax z27.s, p2/M, z27.s, z17.s\n"
-      "fmax z28.s, p2/M, z28.s, z17.s\n"
-      "fmin z29.s, p2/M, z29.s, z16.s\n"
-      "fmax z29.s, p2/M, z29.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmin z27.s, p2/M, z27.s, z17.s\n"
+      "fmin z28.s, p2/M, z28.s, z17.s\n"
+      "fmin z29.s, p2/M, z29.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
+      "fmax z27.s, p2/M, z27.s, z16.s\n"
+      "fmax z28.s, p2/M, z28.s, z16.s\n"
+      "fmax z29.s, p2/M, z29.s, z16.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
-      "st1w { z27.s }, p1, [x24]\n"
-      "st1w { z28.s }, p1, [x23]\n"
-      "st1w { z29.s }, p1, [x22]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
+      "st1w { z27.s }, p1, [x25]\n"
+      "st1w { z28.s }, p1, [x24]\n"
+      "st1w { z29.s }, p1, [x23]\n"
       "78:"  // Height 6: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 67b\n"
       "b 106f\n"
       "79:"  // Height 7
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "80:"  // Height 7: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 81f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 81f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
@@ -1028,20 +1028,20 @@ void sve_hybrid_fp32_mla_8x1VL (
       "b 83f\n"
       "81:"  // Height 7: no bias
       "tbz %x[flags], #0, 82f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z27.s }, p1/Z, [x24]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z28.s }, p1/Z, [x23]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z29.s }, p1/Z, [x22]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z30.s }, p1/Z, [x21]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z27.s }, p1/Z, [x25]\n"
+      "ld1w { z28.s }, p1/Z, [x24]\n"
+      "ld1w { z29.s }, p1/Z, [x23]\n"
+      "ld1w { z30.s }, p1/Z, [x22]\n"
       "b 83f\n"
       "82:"  // Height 7: no accumulate
       "mov z24.b, #0x0\n"
@@ -1052,75 +1052,75 @@ void sve_hybrid_fp32_mla_8x1VL (
       "mov z29.b, #0x0\n"
       "mov z30.b, #0x0\n"
       "83:"  // Height 7: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "84:"  // Height 7: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 85f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "cbnz x9, 86f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "cbnz x10, 86f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 86f\n"
       "85:"  // Height 7: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "86:"  // Height 7: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 88f\n"
       "87:"  // Height 7: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z6.s }, p0/Z, [x22]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
-      "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
-      "add x26, x26, #0x10\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z25.s, z9.s, z1.s[1]\n"
-      "ld1rqw { z6.s }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "fmla z28.s, z8.s, z4.s[0]\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
+      "fmla z28.s, z8.s, z4.s[0]\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
       "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "add x25, x25, #0x10\n"
       "fmla z30.s, z8.s, z6.s[0]\n"
-      "addvl x12, x12, #4\n"
+      "fmla z24.s, z9.s, z0.s[1]\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "fmla z25.s, z9.s, z1.s[1]\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
+      "add x22, x22, #0x10\n"
+      "addvl x12, x12, #4\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "fmla z29.s, z9.s, z5.s[1]\n"
@@ -1141,117 +1141,117 @@ void sve_hybrid_fp32_mla_8x1VL (
       "fmla z30.s, z11.s, z6.s[3]\n"
       "bgt 87b\n"
       "88:"  // Height 7: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
+      "ld1rqw { z6.s }, p0/Z, [x22]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
-      "ld1rqw { z6.s }, p0/Z, [x21]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
       "fmla z30.s, z8.s, z6.s[0]\n"
       "ble 89f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "fmla z29.s, z9.s, z5.s[1]\n"
       "fmla z30.s, z9.s, z6.s[1]\n"
       "ble 89f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z10.s, z4.s[2]\n"
       "fmla z29.s, z10.s, z5.s[2]\n"
       "fmla z30.s, z10.s, z6.s[2]\n"
       "ble 89f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "fmla z28.s, z11.s, z4.s[3]\n"
       "fmla z29.s, z11.s, z5.s[3]\n"
       "fmla z30.s, z11.s, z6.s[3]\n"
       "89:"  // Height 7: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 84b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "tbz %x[flags], #1, 90f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmin z27.s, p2/M, z27.s, z16.s\n"
-      "fmin z28.s, p2/M, z28.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
-      "fmax z27.s, p2/M, z27.s, z17.s\n"
-      "fmax z28.s, p2/M, z28.s, z17.s\n"
-      "fmin z29.s, p2/M, z29.s, z16.s\n"
-      "fmin z30.s, p2/M, z30.s, z16.s\n"
-      "fmax z29.s, p2/M, z29.s, z17.s\n"
-      "fmax z30.s, p2/M, z30.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmin z27.s, p2/M, z27.s, z17.s\n"
+      "fmin z28.s, p2/M, z28.s, z17.s\n"
+      "fmin z29.s, p2/M, z29.s, z17.s\n"
+      "fmin z30.s, p2/M, z30.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
+      "fmax z27.s, p2/M, z27.s, z16.s\n"
+      "fmax z28.s, p2/M, z28.s, z16.s\n"
+      "fmax z29.s, p2/M, z29.s, z16.s\n"
+      "fmax z30.s, p2/M, z30.s, z16.s\n"
       "90:"  // Height 7: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
-      "st1w { z27.s }, p1, [x24]\n"
-      "st1w { z28.s }, p1, [x23]\n"
-      "st1w { z29.s }, p1, [x22]\n"
-      "st1w { z30.s }, p1, [x21]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
+      "st1w { z27.s }, p1, [x25]\n"
+      "st1w { z28.s }, p1, [x24]\n"
+      "st1w { z29.s }, p1, [x23]\n"
+      "st1w { z30.s }, p1, [x22]\n"
       "91:"  // Height 7: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 80b\n"
       "b 106f\n"
       "92:"  // Height 8
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x20\n"
+      "mov x14, %x[bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x11, %x[bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x10, %x[output_ptr]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x20\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "93:"  // Height 8: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p1.s, x19, x13\n"
-      "cbz x11, 94f\n"
-      "ld1w { z24.s }, p2/Z, [x11]\n"
+      "mov x20, #0x0\n"
+      "whilelt p1.s, x20, x13\n"
+      "cbz x14, 94f\n"
+      "ld1w { z24.s }, p2/Z, [x14]\n"
       "mov z25.d, z24.d\n"
-      "addvl x11, x11, #1\n"
       "mov z26.d, z24.d\n"
+      "addvl x14, x14, #1\n"
       "mov z27.d, z24.d\n"
       "mov z28.d, z24.d\n"
       "mov z29.d, z24.d\n"
@@ -1260,22 +1260,22 @@ void sve_hybrid_fp32_mla_8x1VL (
       "b 96f\n"
       "94:"  // Height 8: no bias
       "tbz %x[flags], #0, 95f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z24.s }, p1/Z, [x10]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "ld1w { z25.s }, p1/Z, [x26]\n"
-      "add x25, x26, x19, LSL #2\n"
-      "ld1w { z26.s }, p1/Z, [x25]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "ld1w { z27.s }, p1/Z, [x24]\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z28.s }, p1/Z, [x23]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z29.s }, p1/Z, [x22]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z30.s }, p1/Z, [x21]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z31.s }, p1/Z, [x20]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "ld1w { z24.s }, p1/Z, [x11]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z25.s }, p1/Z, [x27]\n"
+      "ld1w { z26.s }, p1/Z, [x26]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z27.s }, p1/Z, [x25]\n"
+      "ld1w { z28.s }, p1/Z, [x24]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z29.s }, p1/Z, [x23]\n"
+      "ld1w { z30.s }, p1/Z, [x22]\n"
+      "ld1w { z31.s }, p1/Z, [x21]\n"
       "b 96f\n"
       "95:"  // Height 8: no accumulate
       "mov z24.b, #0x0\n"
@@ -1287,80 +1287,80 @@ void sve_hybrid_fp32_mla_8x1VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "96:"  // Height 8: setup done
-      "mov x9, #0x0\n"
+      "mov x10, #0x0\n"
       "97:"  // Height 8: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w28, [x20, x9, LSL #0x2]\n"
+      "ldr w9, [x20, x10, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 98f\n"
-      "ldr x20, [%x[input_ptr], x9, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x27, [x20, #0x0]\n"
-      "ldr x26, [x20, #0x8]\n"
-      "ldr x25, [x20, #0x10]\n"
-      "ldr x24, [x20, #0x18]\n"
-      "ldr x23, [x20, #0x20]\n"
-      "ldr x22, [x20, #0x28]\n"
-      "ldr x21, [x20, #0x30]\n"
-      "ldr x20, [x20, #0x38]\n"
-      "cbnz x9, 99f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x27, x27, x19, LSL #2\n"
-      "add x26, x26, x19, LSL #2\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x28, [x21, #0x0]\n"
+      "ldr x27, [x21, #0x8]\n"
+      "ldr x26, [x21, #0x10]\n"
+      "ldr x25, [x21, #0x18]\n"
+      "ldr x24, [x21, #0x20]\n"
+      "ldr x23, [x21, #0x28]\n"
+      "ldr x22, [x21, #0x30]\n"
+      "ldr x21, [x21, #0x38]\n"
+      "cbnz x10, 99f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x28, x28, x20, LSL #2\n"
+      "add x27, x27, x20, LSL #2\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 99f\n"
       "98:"  // Height 8: setup direct input
-      "mov x27, %x[input_ptr]\n"
-      "add x26, x27, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x28, %x[input_ptr]\n"
+      "add x27, x28, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "99:"  // Height 8: input setup done
-      "cmp x28, #0x4\n"
+      "cmp x9, #0x4\n"
       "ble 101f\n"
       "100:"  // Height 8: Multiply loop: Main loop head
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "sub x9, x9, #0x4\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "cmp x9, #0x4\n"
+      "add x28, x28, #0x10\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
+      "add x27, x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqw { z6.s }, p0/Z, [x22]\n"
+      "ld1rqw { z7.s }, p0/Z, [x21]\n"
+      "add x25, x25, #0x10\n"
+      "add x24, x24, #0x10\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
-      "sub x28, x28, #0x4\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "cmp x28, #0x4\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "add x27, x27, #0x10\n"
-      "fmla z24.s, z9.s, z0.s[1]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
-      "add x26, x26, #0x10\n"
+      "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      "fmla z25.s, z9.s, z1.s[1]\n"
-      "ld1rqw { z6.s }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
+      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
-      "ld1rqw { z7.s }, p0/Z, [x20]\n"
-      "add x22, x22, #0x10\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
-      "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "fmla z30.s, z8.s, z6.s[0]\n"
-      "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "add x20, x20, #0x10\n"
       "fmla z31.s, z8.s, z7.s[0]\n"
+      "add x21, x21, #0x10\n"
       "addvl x12, x12, #4\n"
+      "fmla z24.s, z9.s, z0.s[1]\n"
+      "fmla z25.s, z9.s, z1.s[1]\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
@@ -1385,46 +1385,46 @@ void sve_hybrid_fp32_mla_8x1VL (
       "fmla z31.s, z11.s, z7.s[3]\n"
       "bgt 100b\n"
       "101:"  // Height 8: Multiply loop: Single iteration only
+      "whilelt p0.s, XZR, x9\n"
+      "ld1rqw { z0.s }, p0/Z, [x28]\n"
+      "ld1rqw { z1.s }, p0/Z, [x27]\n"
+      "subs x9, x9, #0x1\n"
+      "ld1rqw { z2.s }, p0/Z, [x26]\n"
+      "ld1rqw { z3.s }, p0/Z, [x25]\n"
+      "ld1rqw { z4.s }, p0/Z, [x24]\n"
+      "ld1rqw { z5.s }, p0/Z, [x23]\n"
+      "ld1rqw { z6.s }, p0/Z, [x22]\n"
+      "ld1rqw { z7.s }, p0/Z, [x21]\n"
       "ld1w { z8.s }, p2/Z, [x12]\n"
-      "whilelt p0.s, XZR, x28\n"
-      "subs x28, x28, #0x1\n"
-      "ld1rqw { z0.s }, p0/Z, [x27]\n"
       "fmla z24.s, z8.s, z0.s[0]\n"
-      "ld1rqw { z1.s }, p0/Z, [x26]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z8.s, z1.s[0]\n"
-      "ld1rqw { z2.s }, p0/Z, [x25]\n"
-      "ld1rqw { z3.s }, p0/Z, [x24]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z8.s, z2.s[0]\n"
-      "ld1rqw { z4.s }, p0/Z, [x23]\n"
       "fmla z27.s, z8.s, z3.s[0]\n"
-      "ld1rqw { z5.s }, p0/Z, [x22]\n"
-      "ld1rqw { z6.s }, p0/Z, [x21]\n"
       "fmla z28.s, z8.s, z4.s[0]\n"
-      "ld1rqw { z7.s }, p0/Z, [x20]\n"
       "fmla z29.s, z8.s, z5.s[0]\n"
       "fmla z30.s, z8.s, z6.s[0]\n"
       "fmla z31.s, z8.s, z7.s[0]\n"
       "ble 102f\n"
       "ld1w { z9.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z9.s, z0.s[1]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z9.s, z1.s[1]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z9.s, z2.s[1]\n"
       "fmla z27.s, z9.s, z3.s[1]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z9.s, z4.s[1]\n"
       "fmla z29.s, z9.s, z5.s[1]\n"
       "fmla z30.s, z9.s, z6.s[1]\n"
       "fmla z31.s, z9.s, z7.s[1]\n"
       "ble 102f\n"
       "ld1w { z10.s }, p2/Z, [x12]\n"
+      "subs x9, x9, #0x1\n"
       "fmla z24.s, z10.s, z0.s[2]\n"
-      "subs x28, x28, #0x1\n"
       "fmla z25.s, z10.s, z1.s[2]\n"
-      "addvl x12, x12, #1\n"
       "fmla z26.s, z10.s, z2.s[2]\n"
       "fmla z27.s, z10.s, z3.s[2]\n"
+      "addvl x12, x12, #1\n"
       "fmla z28.s, z10.s, z4.s[2]\n"
       "fmla z29.s, z10.s, z5.s[2]\n"
       "fmla z30.s, z10.s, z6.s[2]\n"
@@ -1432,8 +1432,8 @@ void sve_hybrid_fp32_mla_8x1VL (
       "ble 102f\n"
       "ld1w { z11.s }, p2/Z, [x12]\n"
       "fmla z24.s, z11.s, z0.s[3]\n"
-      "addvl x12, x12, #1\n"
       "fmla z25.s, z11.s, z1.s[3]\n"
+      "addvl x12, x12, #1\n"
       "fmla z26.s, z11.s, z2.s[3]\n"
       "fmla z27.s, z11.s, z3.s[3]\n"
       "fmla z28.s, z11.s, z4.s[3]\n"
@@ -1441,69 +1441,69 @@ void sve_hybrid_fp32_mla_8x1VL (
       "fmla z30.s, z11.s, z6.s[3]\n"
       "fmla z31.s, z11.s, z7.s[3]\n"
       "102:"  // Height 8: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x9, x9, #0x1\n"
-      "cmp x9, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x10, x10, #0x1\n"
+      "cmp x10, x20\n"
       "bne 97b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x26, x10, x19, LSL #2\n"
-      "add x25, x26, x19, LSL #2\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x27, x11, x20, LSL #2\n"
+      "add x26, x27, x20, LSL #2\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "tbz %x[flags], #1, 103f\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z17.s }, p2/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z16.s }, p2/Z, [x19]\n"
-      "fmin z24.s, p2/M, z24.s, z16.s\n"
-      "fmin z25.s, p2/M, z25.s, z16.s\n"
-      "fmin z26.s, p2/M, z26.s, z16.s\n"
-      "fmin z27.s, p2/M, z27.s, z16.s\n"
-      "fmin z28.s, p2/M, z28.s, z16.s\n"
-      "fmax z24.s, p2/M, z24.s, z17.s\n"
-      "fmax z25.s, p2/M, z25.s, z17.s\n"
-      "fmax z26.s, p2/M, z26.s, z17.s\n"
-      "fmax z27.s, p2/M, z27.s, z17.s\n"
-      "fmax z28.s, p2/M, z28.s, z17.s\n"
-      "fmin z29.s, p2/M, z29.s, z16.s\n"
-      "fmin z30.s, p2/M, z30.s, z16.s\n"
-      "fmin z31.s, p2/M, z31.s, z16.s\n"
-      "fmax z29.s, p2/M, z29.s, z17.s\n"
-      "fmax z30.s, p2/M, z30.s, z17.s\n"
-      "fmax z31.s, p2/M, z31.s, z17.s\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z17.s }, p2/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z16.s }, p2/Z, [x20]\n"
+      "fmin z24.s, p2/M, z24.s, z17.s\n"
+      "fmin z25.s, p2/M, z25.s, z17.s\n"
+      "fmin z26.s, p2/M, z26.s, z17.s\n"
+      "fmin z27.s, p2/M, z27.s, z17.s\n"
+      "fmin z28.s, p2/M, z28.s, z17.s\n"
+      "fmin z29.s, p2/M, z29.s, z17.s\n"
+      "fmin z30.s, p2/M, z30.s, z17.s\n"
+      "fmin z31.s, p2/M, z31.s, z17.s\n"
+      "fmax z24.s, p2/M, z24.s, z16.s\n"
+      "fmax z25.s, p2/M, z25.s, z16.s\n"
+      "fmax z26.s, p2/M, z26.s, z16.s\n"
+      "fmax z27.s, p2/M, z27.s, z16.s\n"
+      "fmax z28.s, p2/M, z28.s, z16.s\n"
+      "fmax z29.s, p2/M, z29.s, z16.s\n"
+      "fmax z30.s, p2/M, z30.s, z16.s\n"
+      "fmax z31.s, p2/M, z31.s, z16.s\n"
       "103:"  // Height 8: No activation
-      "st1w { z24.s }, p1, [x10]\n"
-      "addvl x10, x10, #1\n"
-      "st1w { z25.s }, p1, [x26]\n"
-      "st1w { z26.s }, p1, [x25]\n"
-      "st1w { z27.s }, p1, [x24]\n"
-      "st1w { z28.s }, p1, [x23]\n"
-      "st1w { z29.s }, p1, [x22]\n"
-      "st1w { z30.s }, p1, [x21]\n"
-      "st1w { z31.s }, p1, [x20]\n"
+      "st1w { z24.s }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
+      "st1w { z25.s }, p1, [x27]\n"
+      "st1w { z26.s }, p1, [x26]\n"
+      "st1w { z27.s }, p1, [x25]\n"
+      "st1w { z28.s }, p1, [x24]\n"
+      "st1w { z29.s }, p1, [x23]\n"
+      "st1w { z30.s }, p1, [x22]\n"
+      "st1w { z31.s }, p1, [x21]\n"
       "104:"  // Height 8: Writeback done
       "decw x13\n"
       "cmp x13, XZR\n"
       "bgt 93b\n"
       "subs %x[M], %x[M], #0x8\n"
       "beq 106f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 105f\n"
-      "add x20, x20, #0x8\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x8\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "105:"  // Update direct input
-      "mov x19, #0x20\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x20\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "106:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
index 43b0f54805..8d05c1ffb3 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -100,39 +100,39 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
-      "mov x9, %x[bias]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x27, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x10, %x[bias]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x28\n"
-      "cbz x9, 3f\n"
-      "ld1w { z8.s }, p7/Z, [x9]\n"
-      "ld1w { z9.s }, p7/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p7/Z, [x9, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x9\n"
+      "cbz x10, 3f\n"
+      "ld1w { z8.s }, p7/Z, [x10]\n"
+      "ld1w { z9.s }, p7/Z, [x10, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p7/Z, [x9, #3, MUL VL]\n"
-      "ld1w { z12.s }, p7/Z, [x9, #4, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x10, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z13.s }, p7/Z, [x9, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x10, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x10, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x9, x9, #6\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "addvl x10, x10, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -140,16 +140,16 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z9.s }, p6/Z, [x26]\n"
-      "ld1w { z10.s }, p5/Z, [x26, #1, MUL VL]\n"
+      "ld1w { z9.s }, p6/Z, [x27]\n"
+      "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z14.d\n"
-      "ld1w { z11.s }, p4/Z, [x26, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x26, #3, MUL VL]\n"
       "zip2 z14.d, z9.d, z14.d\n"
+      "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
-      "ld1w { z13.s }, p2/Z, [x26, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x26, #5, MUL VL]\n"
       "zip2 z15.d, z10.d, z15.d\n"
+      "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
       "zip1 z11.d, z12.d, z17.d\n"
@@ -173,94 +173,94 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "6:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
-      "sub x24, x24, #0x4\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
+      "sub x25, x25, #0x4\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
-      "add x23, x23, #0x10\n"
-      "addvl x27, x27, #-4\n"
+      "add x24, x24, #0x10\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 6b\n"
       "uzp1 z8.d, z8.d, z14.d\n"
       "uzp1 z9.d, z9.d, z15.d\n"
@@ -269,10 +269,10 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z12.d, z12.d, z18.d\n"
       "uzp1 z13.d, z13.d, z19.d\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z8.s, p7/M, z8.s, z1.s\n"
       "fmin z9.s, p7/M, z9.s, z1.s\n"
       "fmin z10.s, p7/M, z10.s, z1.s\n"
@@ -286,52 +286,52 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z12.s, p7/M, z12.s, z0.s\n"
       "fmax z13.s, p7/M, z13.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p6, [x26]\n"
-      "st1w { z9.s }, p5, [x26, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x26, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x26, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x26, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x26, #5, MUL VL]\n"
-      "addvl x26, x26, #6\n"
+      "st1w { z8.s }, p6, [x27]\n"
+      "st1w { z9.s }, p5, [x27, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x27, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x27, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x27, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x27, #5, MUL VL]\n"
+      "addvl x27, x27, #6\n"
       "13:"  // Height 1: Writeback done
-      "decw x28, ALL, MUL #6\n"
-      "cmp x28, XZR\n"
+      "decw x9, ALL, MUL #6\n"
+      "cmp x9, XZR\n"
       "bgt 2b\n"
       "b 54f\n"
       "14:"  // Height 2
-      "mov x9, %x[bias]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x27, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x10, %x[bias]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x28\n"
-      "cbz x9, 16f\n"
-      "ld1w { z8.s }, p7/Z, [x9]\n"
-      "ld1w { z9.s }, p7/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p7/Z, [x9, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x9\n"
+      "cbz x10, 16f\n"
+      "ld1w { z8.s }, p7/Z, [x10]\n"
+      "ld1w { z9.s }, p7/Z, [x10, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p7/Z, [x9, #3, MUL VL]\n"
-      "ld1w { z12.s }, p7/Z, [x9, #4, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x10, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z13.s }, p7/Z, [x9, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x10, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x10, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x9, x9, #6\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "addvl x10, x10, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -339,26 +339,26 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x26]\n"
-      "ld1w { z10.s }, p5/Z, [x26, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x26, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x26, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x26, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x26, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x27]\n"
+      "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
       "zip1 z12.d, z13.d, z18.d\n"
@@ -380,111 +380,111 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "19:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
       "21:"  // Height 2: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
-      "sub x24, x24, #0x4\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
+      "sub x25, x25, #0x4\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6466e40d  // bfmmla z13.s, z0.h, z6.h\n"
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
-      "add x22, x26, x19, LSL #2\n"
+      "add x23, x27, x20, LSL #2\n"
       "uzp1 z14.d, z9.d, z15.d\n"
       "uzp2 z9.d, z9.d, z15.d\n"
       "uzp1 z15.d, z10.d, z16.d\n"
@@ -496,10 +496,10 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z18.d, z13.d, z19.d\n"
       "uzp2 z13.d, z13.d, z19.d\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -525,58 +525,58 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z12.s, p7/M, z12.s, z0.s\n"
       "fmax z13.s, p7/M, z13.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z4.s }, p6, [x26]\n"
-      "st1w { z14.s }, p5, [x26, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x26, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x26, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x26, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x26, #5, MUL VL]\n"
-      "addvl x26, x26, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x27]\n"
+      "st1w { z14.s }, p5, [x27, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x27, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x27, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x27, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x27, #5, MUL VL]\n"
+      "addvl x27, x27, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
       "26:"  // Height 2: Writeback done
-      "decw x28, ALL, MUL #6\n"
-      "cmp x28, XZR\n"
+      "decw x9, ALL, MUL #6\n"
+      "cmp x9, XZR\n"
       "bgt 15b\n"
       "b 54f\n"
       "27:"  // Height 3
-      "mov x9, %x[bias]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x27, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "mov x10, %x[bias]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x28\n"
-      "cbz x9, 29f\n"
-      "ld1w { z8.s }, p7/Z, [x9]\n"
-      "ld1w { z9.s }, p7/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p7/Z, [x9, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x9\n"
+      "cbz x10, 29f\n"
+      "ld1w { z8.s }, p7/Z, [x10]\n"
+      "ld1w { z9.s }, p7/Z, [x10, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p7/Z, [x9, #3, MUL VL]\n"
-      "ld1w { z12.s }, p7/Z, [x9, #4, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x10, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z13.s }, p7/Z, [x9, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x10, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x10, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x9, x9, #6\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "addvl x10, x10, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -596,39 +596,39 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x26]\n"
-      "ld1w { z10.s }, p5/Z, [x26, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x26, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x26, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x26, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x26, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x27]\n"
+      "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z21.s }, p6/Z, [x21]\n"
-      "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
+      "ld1w { z21.s }, p6/Z, [x22]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
-      "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n"
       "zip1 z12.d, z13.d, z18.d\n"
       "zip2 z18.d, z13.d, z18.d\n"
-      "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n"
-      "ld1w { z4.s }, p1/Z, [x21, #5, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n"
       "zip1 z13.d, z20.d, z19.d\n"
       "zip2 z19.d, z20.d, z19.d\n"
+      "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n"
       "zip1 z20.d, z21.d, z26.d\n"
       "zip2 z26.d, z21.d, z26.d\n"
       "zip1 z21.d, z22.d, z27.d\n"
@@ -668,78 +668,78 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "32:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "34:"  // Height 3: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
-      "sub x24, x24, #0x4\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
+      "sub x25, x25, #0x4\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
-      "add x22, x22, #0x10\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -749,48 +749,48 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -799,17 +799,17 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
       "uzp1 z14.d, z9.d, z15.d\n"
       "uzp2 z9.d, z9.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 z15.d, z10.d, z16.d\n"
       "uzp2 z10.d, z10.d, z16.d\n"
       "uzp1 z16.d, z11.d, z17.d\n"
@@ -825,10 +825,10 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z24.d, z24.d, z30.d\n"
       "uzp1 z25.d, z25.d, z31.d\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -866,67 +866,67 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z24.s, p7/M, z24.s, z0.s\n"
       "fmax z25.s, p7/M, z25.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z4.s }, p6, [x26]\n"
-      "st1w { z14.s }, p5, [x26, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x26, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x26, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x26, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x26, #5, MUL VL]\n"
-      "addvl x26, x26, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
-      "st1w { z20.s }, p6, [x21]\n"
-      "st1w { z21.s }, p5, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p4, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p3, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p2, [x21, #4, MUL VL]\n"
-      "st1w { z25.s }, p1, [x21, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x27]\n"
+      "st1w { z14.s }, p5, [x27, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x27, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x27, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x27, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x27, #5, MUL VL]\n"
+      "addvl x27, x27, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
+      "st1w { z20.s }, p6, [x22]\n"
+      "st1w { z21.s }, p5, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p4, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p3, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p2, [x22, #4, MUL VL]\n"
+      "st1w { z25.s }, p1, [x22, #5, MUL VL]\n"
       "39:"  // Height 3: Writeback done
-      "decw x28, ALL, MUL #6\n"
-      "cmp x28, XZR\n"
+      "decw x9, ALL, MUL #6\n"
+      "cmp x9, XZR\n"
       "bgt 28b\n"
       "b 54f\n"
       "40:"  // Height 4
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x10\n"
-      "mov x9, %x[bias]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x27, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x26, %x[output_ptr]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x10\n"
+      "mov x10, %x[bias]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p6.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p5.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p4.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x28\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x28\n"
-      "cbz x9, 42f\n"
-      "ld1w { z8.s }, p7/Z, [x9]\n"
-      "ld1w { z9.s }, p7/Z, [x9, #1, MUL VL]\n"
-      "ld1w { z10.s }, p7/Z, [x9, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p6.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p5.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p4.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x9\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x9\n"
+      "cbz x10, 42f\n"
+      "ld1w { z8.s }, p7/Z, [x10]\n"
+      "ld1w { z9.s }, p7/Z, [x10, #1, MUL VL]\n"
       "zip2 z14.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p7/Z, [x9, #3, MUL VL]\n"
-      "ld1w { z12.s }, p7/Z, [x9, #4, MUL VL]\n"
+      "ld1w { z10.s }, p7/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z11.s }, p7/Z, [x10, #3, MUL VL]\n"
       "zip2 z15.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "ld1w { z13.s }, p7/Z, [x9, #5, MUL VL]\n"
+      "ld1w { z12.s }, p7/Z, [x10, #4, MUL VL]\n"
+      "ld1w { z13.s }, p7/Z, [x10, #5, MUL VL]\n"
       "zip2 z16.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
-      "addvl x9, x9, #6\n"
       "zip2 z17.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
+      "addvl x10, x10, #6\n"
       "zip2 z18.d, z12.d, z12.d\n"
       "zip1 z12.d, z12.d, z12.d\n"
       "zip2 z19.d, z13.d, z13.d\n"
@@ -946,52 +946,52 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z9.s }, p6/Z, [x26]\n"
-      "ld1w { z10.s }, p5/Z, [x26, #1, MUL VL]\n"
-      "ld1w { z11.s }, p4/Z, [x26, #2, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x26, #3, MUL VL]\n"
-      "ld1w { z13.s }, p2/Z, [x26, #4, MUL VL]\n"
-      "ld1w { z20.s }, p1/Z, [x26, #5, MUL VL]\n"
-      "ld1w { z14.s }, p6/Z, [x22]\n"
-      "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p6/Z, [x27]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n"
+      "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n"
+      "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n"
+      "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n"
+      "ld1w { z14.s }, p6/Z, [x23]\n"
       "zip1 z8.d, z9.d, z14.d\n"
       "zip2 z14.d, z9.d, z14.d\n"
-      "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z15.d\n"
       "zip2 z15.d, z10.d, z15.d\n"
-      "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n"
       "zip1 z10.d, z11.d, z16.d\n"
       "zip2 z16.d, z11.d, z16.d\n"
-      "ld1w { z21.s }, p6/Z, [x21]\n"
-      "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n"
+      "ld1w { z21.s }, p6/Z, [x22]\n"
       "zip1 z11.d, z12.d, z17.d\n"
       "zip2 z17.d, z12.d, z17.d\n"
-      "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n"
       "zip1 z12.d, z13.d, z18.d\n"
       "zip2 z18.d, z13.d, z18.d\n"
-      "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n"
-      "ld1w { z4.s }, p1/Z, [x21, #5, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n"
       "zip1 z13.d, z20.d, z19.d\n"
       "zip2 z19.d, z20.d, z19.d\n"
-      "ld1w { z26.s }, p6/Z, [x20]\n"
-      "ld1w { z27.s }, p5/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n"
+      "ld1w { z26.s }, p6/Z, [x21]\n"
       "zip1 z20.d, z21.d, z26.d\n"
       "zip2 z26.d, z21.d, z26.d\n"
-      "ld1w { z28.s }, p4/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z29.s }, p3/Z, [x20, #3, MUL VL]\n"
+      "ld1w { z27.s }, p5/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21, #2, MUL VL]\n"
       "zip1 z21.d, z22.d, z27.d\n"
       "zip2 z27.d, z22.d, z27.d\n"
-      "ld1w { z30.s }, p2/Z, [x20, #4, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x20, #5, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #4, MUL VL]\n"
       "zip1 z22.d, z23.d, z28.d\n"
       "zip2 z28.d, z23.d, z28.d\n"
+      "ld1w { z31.s }, p1/Z, [x21, #5, MUL VL]\n"
       "zip1 z23.d, z24.d, z29.d\n"
       "zip2 z29.d, z24.d, z29.d\n"
       "zip1 z24.d, z25.d, z30.d\n"
@@ -1025,86 +1025,86 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "45:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w24, [x19, x25, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "47:"  // Height 4: input setup done
-      "cmp x24, #0x4\n"
+      "cmp x25, #0x4\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
-      "ld1rqw { z3.s }, p0/Z, [x20]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
+      "ld1rqw { z3.s }, p0/Z, [x21]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
       ".inst 0x658abc63  // bfcvt z3.h, p7/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
-      "sub x24, x24, #0x4\n"
+      "sub x25, x25, #0x4\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
-      "cmp x24, #0x4\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
+      "cmp x25, #0x4\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
-      "add x22, x22, #0x10\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
-      "add x20, x20, #0x10\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -1114,52 +1114,52 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x24\n"
-      "ld1rqw { z0.s }, p0/Z, [x23]\n"
-      "ld1rqw { z1.s }, p0/Z, [x22]\n"
+      "whilelt p0.s, XZR, x25\n"
+      "ld1rqw { z0.s }, p0/Z, [x24]\n"
+      "ld1rqw { z1.s }, p0/Z, [x23]\n"
       ".inst 0x658abc00  // bfcvt z0.h, p7/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x21]\n"
-      "ld1rqw { z3.s }, p0/Z, [x20]\n"
+      "ld1rqw { z2.s }, p0/Z, [x22]\n"
+      "ld1rqw { z3.s }, p0/Z, [x21]\n"
       ".inst 0x658abc21  // bfcvt z1.h, p7/M, z1.s\n"
       ".inst 0x658abc42  // bfcvt z2.h, p7/M, z2.s\n"
       ".inst 0x658abc63  // bfcvt z3.h, p7/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z4.h }, p7/Z, [x27]\n"
-      "ld1h { z5.h }, p7/Z, [x27, #1, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #2, MUL VL]\n"
-      "ld1h { z7.h }, p7/Z, [x27, #3, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e40e  // bfmmla z14.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #4, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n"
       ".inst 0x6465e45a  // bfmmla z26.s, z2.h, z5.h\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #5, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #6, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n"
       ".inst 0x6467e45b  // bfmmla z27.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #7, MUL VL]\n"
-      "addvl x27, x27, #16\n"
+      "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e410  // bfmmla z16.s, z0.h, z5.h\n"
-      "ld1h { z4.h }, p7/Z, [x27, #-8, MUL VL]\n"
+      "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x6465e45c  // bfmmla z28.s, z2.h, z5.h\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "ld1h { z5.h }, p7/Z, [x27, #-7, MUL VL]\n"
+      "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x6466e457  // bfmmla z23.s, z2.h, z6.h\n"
       ".inst 0x6467e411  // bfmmla z17.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p7/Z, [x27, #-6, MUL VL]\n"
+      "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x6467e45d  // bfmmla z29.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p7/Z, [x27, #-5, MUL VL]\n"
+      "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x6464e40c  // bfmmla z12.s, z0.h, z4.h\n"
-      "addvl x27, x27, #-4\n"
+      "addvl x28, x28, #-4\n"
       ".inst 0x6464e458  // bfmmla z24.s, z2.h, z4.h\n"
       ".inst 0x6465e412  // bfmmla z18.s, z0.h, z5.h\n"
       ".inst 0x6465e45e  // bfmmla z30.s, z2.h, z5.h\n"
@@ -1168,17 +1168,17 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       ".inst 0x6467e413  // bfmmla z19.s, z0.h, z7.h\n"
       ".inst 0x6467e45f  // bfmmla z31.s, z2.h, z7.h\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp1 z4.d, z8.d, z14.d\n"
       "uzp2 z8.d, z8.d, z14.d\n"
       "uzp1 z14.d, z9.d, z15.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z15.d\n"
       "uzp1 z15.d, z10.d, z16.d\n"
       "uzp2 z10.d, z10.d, z16.d\n"
@@ -1201,10 +1201,10 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "uzp1 z30.d, z25.d, z31.d\n"
       "uzp2 z25.d, z25.d, z31.d\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p7/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p7/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p7/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p7/Z, [x20]\n"
       "fmin z4.s, p7/M, z4.s, z1.s\n"
       "fmin z14.s, p7/M, z14.s, z1.s\n"
       "fmin z15.s, p7/M, z15.s, z1.s\n"
@@ -1254,51 +1254,51 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL (
       "fmax z24.s, p7/M, z24.s, z0.s\n"
       "fmax z25.s, p7/M, z25.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z4.s }, p6, [x26]\n"
-      "st1w { z14.s }, p5, [x26, #1, MUL VL]\n"
-      "st1w { z15.s }, p4, [x26, #2, MUL VL]\n"
-      "st1w { z16.s }, p3, [x26, #3, MUL VL]\n"
-      "st1w { z17.s }, p2, [x26, #4, MUL VL]\n"
-      "st1w { z18.s }, p1, [x26, #5, MUL VL]\n"
-      "addvl x26, x26, #6\n"
-      "st1w { z8.s }, p6, [x22]\n"
-      "st1w { z9.s }, p5, [x22, #1, MUL VL]\n"
-      "st1w { z10.s }, p4, [x22, #2, MUL VL]\n"
-      "st1w { z11.s }, p3, [x22, #3, MUL VL]\n"
-      "st1w { z12.s }, p2, [x22, #4, MUL VL]\n"
-      "st1w { z13.s }, p1, [x22, #5, MUL VL]\n"
-      "st1w { z19.s }, p6, [x21]\n"
-      "st1w { z26.s }, p5, [x21, #1, MUL VL]\n"
-      "st1w { z27.s }, p4, [x21, #2, MUL VL]\n"
-      "st1w { z28.s }, p3, [x21, #3, MUL VL]\n"
-      "st1w { z29.s }, p2, [x21, #4, MUL VL]\n"
-      "st1w { z30.s }, p1, [x21, #5, MUL VL]\n"
-      "st1w { z20.s }, p6, [x20]\n"
-      "st1w { z21.s }, p5, [x20, #1, MUL VL]\n"
-      "st1w { z22.s }, p4, [x20, #2, MUL VL]\n"
-      "st1w { z23.s }, p3, [x20, #3, MUL VL]\n"
-      "st1w { z24.s }, p2, [x20, #4, MUL VL]\n"
-      "st1w { z25.s }, p1, [x20, #5, MUL VL]\n"
+      "st1w { z4.s }, p6, [x27]\n"
+      "st1w { z14.s }, p5, [x27, #1, MUL VL]\n"
+      "st1w { z15.s }, p4, [x27, #2, MUL VL]\n"
+      "st1w { z16.s }, p3, [x27, #3, MUL VL]\n"
+      "st1w { z17.s }, p2, [x27, #4, MUL VL]\n"
+      "st1w { z18.s }, p1, [x27, #5, MUL VL]\n"
+      "addvl x27, x27, #6\n"
+      "st1w { z8.s }, p6, [x23]\n"
+      "st1w { z9.s }, p5, [x23, #1, MUL VL]\n"
+      "st1w { z10.s }, p4, [x23, #2, MUL VL]\n"
+      "st1w { z11.s }, p3, [x23, #3, MUL VL]\n"
+      "st1w { z12.s }, p2, [x23, #4, MUL VL]\n"
+      "st1w { z13.s }, p1, [x23, #5, MUL VL]\n"
+      "st1w { z19.s }, p6, [x22]\n"
+      "st1w { z26.s }, p5, [x22, #1, MUL VL]\n"
+      "st1w { z27.s }, p4, [x22, #2, MUL VL]\n"
+      "st1w { z28.s }, p3, [x22, #3, MUL VL]\n"
+      "st1w { z29.s }, p2, [x22, #4, MUL VL]\n"
+      "st1w { z30.s }, p1, [x22, #5, MUL VL]\n"
+      "st1w { z20.s }, p6, [x21]\n"
+      "st1w { z21.s }, p5, [x21, #1, MUL VL]\n"
+      "st1w { z22.s }, p4, [x21, #2, MUL VL]\n"
+      "st1w { z23.s }, p3, [x21, #3, MUL VL]\n"
+      "st1w { z24.s }, p2, [x21, #4, MUL VL]\n"
+      "st1w { z25.s }, p1, [x21, #5, MUL VL]\n"
       "52:"  // Height 4: Writeback done
-      "decw x28, ALL, MUL #6\n"
-      "cmp x28, XZR\n"
+      "decw x9, ALL, MUL #6\n"
+      "cmp x9, XZR\n"
       "bgt 41b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 54f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 53f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "53:"  // Update direct input
-      "mov x19, #0x10\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x10\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "54:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp
index 236eebad66..23d7ff9c3b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -103,42 +103,42 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 3f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 3f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 5f\n"
       "3:"  // Height 1: no bias
       "tbz %x[flags], #0, 4f\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
@@ -156,86 +156,86 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "5:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "6:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 7f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 8f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 8f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
       "b 8f\n"
       "7:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "8:"  // Height 1: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 10f\n"
       "9:"  // Height 1: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x4\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "add x25, x25, #0x10\n"
-      "addvl x9, x9, #8\n"
+      "add x26, x26, #0x10\n"
+      "addvl x10, x10, #8\n"
       "bgt 9b\n"
       "10:"  // Height 1: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       "11:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 6b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
       "uzp1 z9.d, z9.d, z13.d\n"
       "uzp1 z10.d, z10.d, z14.d\n"
       "uzp1 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 12f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z8.s, p5/M, z8.s, z1.s\n"
       "fmin z9.s, p5/M, z9.s, z1.s\n"
       "fmin z10.s, p5/M, z10.s, z1.s\n"
@@ -245,61 +245,61 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z10.s, p5/M, z10.s, z0.s\n"
       "fmax z11.s, p5/M, z11.s, z0.s\n"
       "12:"  // Height 1: No activation
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "13:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "15:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 16f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 16f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "b 18f\n"
       "16:"  // Height 2: no bias
       "tbz %x[flags], #0, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
@@ -315,93 +315,93 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "18:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "19:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 20f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 21f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 21f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
       "b 21f\n"
       "20:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
       "21:"  // Height 2: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 23f\n"
       "22:"  // Height 2: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       "bgt 22b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       "24:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 19b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z6.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "add x24, x28, x19, LSL #2\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
@@ -409,10 +409,10 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
       "tbz %x[flags], #1, 25f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z6.s, p5/M, z6.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -430,46 +430,46 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z10.s, p5/M, z10.s, z0.s\n"
       "fmax z11.s, p5/M, z11.s, z0.s\n"
       "25:"  // Height 2: No activation
-      "st1w { z6.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z6.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
       "26:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "28:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 29f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 29f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -483,29 +483,29 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "b 31f\n"
       "29:"  // Height 3: no bias
       "tbz %x[flags], #0, 30f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
@@ -533,122 +533,122 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "32:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "34:"  // Height 3: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 36f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       "bgt 35b\n"
       "36:"  // Height 3: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "uzp1 z2.h, z2.h, z2.h\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       "37:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
       "uzp1 z6.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "add x23, x24, x19, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
@@ -658,10 +658,10 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z18.d, z18.d, z22.d\n"
       "uzp1 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 38f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z6.s, p5/M, z6.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -687,50 +687,50 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z18.s, p5/M, z18.s, z0.s\n"
       "fmax z19.s, p5/M, z19.s, z0.s\n"
       "38:"  // Height 3: No activation
-      "st1w { z6.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x23]\n"
-      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z6.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x24]\n"
+      "st1w { z17.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x24, #3, MUL VL]\n"
       "39:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "41:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 42f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 42f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -744,38 +744,38 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "b 44f\n"
       "42:"  // Height 4: no bias
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
@@ -799,94 +799,94 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "44:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
       "b 47f\n"
       "46:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "47:"  // Height 4: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 49f\n"
       "48:"  // Height 4: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "add x25, x25, #0x10\n"
+      "add x26, x26, #0x10\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
-      "add x23, x23, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       "bgt 48b\n"
       "49:"  // Height 4: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
       "uzp1 z3.h, z3.h, z3.h\n"
@@ -895,38 +895,38 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       "50:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z6.d, z8.d, z12.d\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
@@ -941,10 +941,10 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z22.d, z19.d, z23.d\n"
       "uzp2 z19.d, z19.d, z23.d\n"
       "tbz %x[flags], #1, 51f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z6.s, p5/M, z6.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -978,54 +978,54 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z18.s, p5/M, z18.s, z0.s\n"
       "fmax z19.s, p5/M, z19.s, z0.s\n"
       "51:"  // Height 4: No activation
-      "st1w { z6.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z6.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "52:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "54:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 55f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 55f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -1047,47 +1047,47 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "b 57f\n"
       "55:"  // Height 5: no bias
       "tbz %x[flags], #0, 56f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
       "zip2 z23.d, z24.d, z23.d\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
@@ -1123,90 +1123,90 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "57:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "58:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 59f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 60f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 60f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
       "b 60f\n"
       "59:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "60:"  // Height 5: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 62f\n"
       "61:"  // Height 5: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x4\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
+      "sub x27, x27, #0x4\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       ".inst 0x658ab484  // bfcvt z4.h, p5/M, z4.s\n"
-      "cmp x26, #0x4\n"
-      "add x25, x25, #0x10\n"
+      "cmp x27, #0x4\n"
+      "add x26, x26, #0x10\n"
       "trn1 z0.d, z0.d, z1.d\n"
       "trn1 z2.d, z2.d, z3.d\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "add x24, x24, #0x10\n"
+      "add x25, x25, #0x10\n"
       "uzp1 z4.h, z4.h, z4.h\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6466e498  // bfmmla z24.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
+      "add x24, x24, #0x10\n"
       "add x23, x23, #0x10\n"
-      "add x22, x22, #0x10\n"
       ".inst 0x6467e49c  // bfmmla z28.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
-      "add x21, x21, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6466e499  // bfmmla z25.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
       ".inst 0x6467e49d  // bfmmla z29.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6466e49a  // bfmmla z26.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
       ".inst 0x6467e49e  // bfmmla z30.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6466e49b  // bfmmla z27.s, z4.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
@@ -1214,21 +1214,21 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       ".inst 0x6467e49f  // bfmmla z31.s, z4.h, z7.h\n"
       "bgt 61b\n"
       "62:"  // Height 5: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       ".inst 0x658ab484  // bfcvt z4.h, p5/M, z4.s\n"
       "trn1 z0.d, z0.d, z1.d\n"
@@ -1237,47 +1237,47 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z4.h, z4.h, z4.h\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6466e498  // bfmmla z24.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
       ".inst 0x6467e49c  // bfmmla z28.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6466e499  // bfmmla z25.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
       ".inst 0x6467e49d  // bfmmla z29.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6466e49a  // bfmmla z26.s, z4.h, z6.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
       ".inst 0x6467e49e  // bfmmla z30.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
-      "addvl x9, x9, #8\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6466e49b  // bfmmla z27.s, z4.h, z6.h\n"
       ".inst 0x6467e40f  // bfmmla z15.s, z0.h, z7.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       ".inst 0x6467e49f  // bfmmla z31.s, z4.h, z7.h\n"
       "63:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 58b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z6.d, z8.d, z12.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
@@ -1296,10 +1296,10 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z26.d, z26.d, z30.d\n"
       "uzp1 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 64f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z6.s, p5/M, z6.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -1341,61 +1341,61 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z26.s, p5/M, z26.s, z0.s\n"
       "fmax z27.s, p5/M, z27.s, z0.s\n"
       "64:"  // Height 5: No activation
-      "st1w { z6.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x21]\n"
-      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z6.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x22]\n"
+      "st1w { z25.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x22, #3, MUL VL]\n"
       "65:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "mov x11, %x[bias]\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "mov x12, %x[bias]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "67:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "cbz x11, 68f\n"
-      "ld1w { z8.s }, p5/Z, [x11]\n"
-      "ld1w { z9.s }, p5/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z10.s }, p5/Z, [x11, #2, MUL VL]\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "cbz x12, 68f\n"
+      "ld1w { z8.s }, p5/Z, [x12]\n"
+      "ld1w { z9.s }, p5/Z, [x12, #1, MUL VL]\n"
       "zip2 z12.d, z8.d, z8.d\n"
       "zip1 z8.d, z8.d, z8.d\n"
-      "ld1w { z11.s }, p5/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z10.s }, p5/Z, [x12, #2, MUL VL]\n"
+      "ld1w { z11.s }, p5/Z, [x12, #3, MUL VL]\n"
       "zip2 z13.d, z9.d, z9.d\n"
       "zip1 z9.d, z9.d, z9.d\n"
-      "addvl x11, x11, #4\n"
       "zip2 z14.d, z10.d, z10.d\n"
       "zip1 z10.d, z10.d, z10.d\n"
+      "addvl x12, x12, #4\n"
       "zip2 z15.d, z11.d, z11.d\n"
       "zip1 z11.d, z11.d, z11.d\n"
       "mov z16.d, z8.d\n"
@@ -1417,55 +1417,55 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "b 70f\n"
       "68:"  // Height 6: no bias
       "tbz %x[flags], #0, 69f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x24]\n"
-      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x25]\n"
       "zip1 z8.d, z9.d, z12.d\n"
+      "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
+      "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x24]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x23]\n"
-      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
+      "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
+      "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x23]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x22]\n"
-      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
+      "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
+      "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x22]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z25.s }, p4/Z, [x21]\n"
-      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
+      "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
+      "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x21]\n"
       "zip2 z23.d, z24.d, z23.d\n"
-      "ld1w { z28.s }, p4/Z, [x20]\n"
-      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
+      "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip2 z28.d, z25.d, z28.d\n"
-      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip1 z25.d, z26.d, z29.d\n"
+      "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z29.d, z26.d, z29.d\n"
       "zip1 z26.d, z27.d, z30.d\n"
       "zip2 z30.d, z27.d, z30.d\n"
@@ -1498,97 +1498,97 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "mov z30.b, #0x0\n"
       "mov z31.b, #0x0\n"
       "70:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "71:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 72f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 73f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19, LSL #2\n"
-      "add x24, x24, x19, LSL #2\n"
-      "add x23, x23, x19, LSL #2\n"
-      "add x22, x22, x19, LSL #2\n"
-      "add x21, x21, x19, LSL #2\n"
-      "add x20, x20, x19, LSL #2\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 73f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20, LSL #2\n"
+      "add x25, x25, x20, LSL #2\n"
+      "add x24, x24, x20, LSL #2\n"
+      "add x23, x23, x20, LSL #2\n"
+      "add x22, x22, x20, LSL #2\n"
+      "add x21, x21, x20, LSL #2\n"
       "b 73f\n"
       "72:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "73:"  // Height 6: input setup done
-      "cmp x26, #0x4\n"
+      "cmp x27, #0x4\n"
       "ble 75f\n"
       "74:"  // Height 6: Multiply loop: Main loop head
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       ".inst 0x658ab484  // bfcvt z4.h, p5/M, z4.s\n"
       ".inst 0x658ab4a5  // bfcvt z5.h, p5/M, z5.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
-      "sub x26, x26, #0x4\n"
-      "cmp x26, #0x4\n"
+      "sub x27, x27, #0x4\n"
+      "cmp x27, #0x4\n"
       "uzp1 z3.h, z3.h, z3.h\n"
       "uzp1 z4.h, z4.h, z4.h\n"
+      "add x26, x26, #0x10\n"
       "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "uzp1 z5.h, z5.h, z5.h\n"
       "trn1 z0.d, z0.d, z1.d\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      "add x23, x23, #0x10\n"
+      "add x24, x24, #0x10\n"
       "trn1 z2.d, z2.d, z3.d\n"
       "trn1 z4.d, z4.d, z5.d\n"
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
-      "add x22, x22, #0x10\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x6466e498  // bfmmla z24.s, z4.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
       ".inst 0x6467e49c  // bfmmla z28.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
-      "add x20, x20, #0x10\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6466e499  // bfmmla z25.s, z4.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
       ".inst 0x6467e49d  // bfmmla z29.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6466e49a  // bfmmla z26.s, z4.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
       ".inst 0x6467e49e  // bfmmla z30.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6466e49b  // bfmmla z27.s, z4.h, z6.h\n"
@@ -1597,22 +1597,22 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       ".inst 0x6467e49f  // bfmmla z31.s, z4.h, z7.h\n"
       "bgt 74b\n"
       "75:"  // Height 6: Multiply loop: Single iteration only
-      "whilelt p0.s, XZR, x26\n"
-      "ld1rqw { z0.s }, p0/Z, [x25]\n"
-      "ld1rqw { z1.s }, p0/Z, [x24]\n"
+      "whilelt p0.s, XZR, x27\n"
+      "ld1rqw { z0.s }, p0/Z, [x26]\n"
+      "ld1rqw { z1.s }, p0/Z, [x25]\n"
       ".inst 0x658ab400  // bfcvt z0.h, p5/M, z0.s\n"
-      "ld1rqw { z2.s }, p0/Z, [x23]\n"
-      "ld1rqw { z3.s }, p0/Z, [x22]\n"
+      "ld1rqw { z2.s }, p0/Z, [x24]\n"
+      "ld1rqw { z3.s }, p0/Z, [x23]\n"
       ".inst 0x658ab421  // bfcvt z1.h, p5/M, z1.s\n"
       ".inst 0x658ab442  // bfcvt z2.h, p5/M, z2.s\n"
-      "ld1rqw { z4.s }, p0/Z, [x21]\n"
-      "ld1rqw { z5.s }, p0/Z, [x20]\n"
+      "ld1rqw { z4.s }, p0/Z, [x22]\n"
+      "ld1rqw { z5.s }, p0/Z, [x21]\n"
       ".inst 0x658ab463  // bfcvt z3.h, p5/M, z3.s\n"
       ".inst 0x658ab484  // bfcvt z4.h, p5/M, z4.s\n"
       ".inst 0x658ab4a5  // bfcvt z5.h, p5/M, z5.s\n"
       "uzp1 z0.h, z0.h, z0.h\n"
-      "ld1h { z6.h }, p5/Z, [x9]\n"
-      "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n"
       "uzp1 z1.h, z1.h, z1.h\n"
       "uzp1 z2.h, z2.h, z2.h\n"
       "uzp1 z3.h, z3.h, z3.h\n"
@@ -1625,27 +1625,27 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       ".inst 0x6466e450  // bfmmla z16.s, z2.h, z6.h\n"
       ".inst 0x6466e498  // bfmmla z24.s, z4.h, z6.h\n"
       ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x6467e454  // bfmmla z20.s, z2.h, z7.h\n"
       ".inst 0x6467e49c  // bfmmla z28.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
       ".inst 0x6466e451  // bfmmla z17.s, z2.h, z6.h\n"
       ".inst 0x6466e499  // bfmmla z25.s, z4.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x6467e455  // bfmmla z21.s, z2.h, z7.h\n"
       ".inst 0x6467e49d  // bfmmla z29.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6466e452  // bfmmla z18.s, z2.h, z6.h\n"
       ".inst 0x6466e49a  // bfmmla z26.s, z4.h, z6.h\n"
       ".inst 0x6467e40e  // bfmmla z14.s, z0.h, z7.h\n"
-      "ld1h { z6.h }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x6467e456  // bfmmla z22.s, z2.h, z7.h\n"
       ".inst 0x6467e49e  // bfmmla z30.s, z4.h, z7.h\n"
-      "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x6466e40b  // bfmmla z11.s, z0.h, z6.h\n"
       ".inst 0x6466e453  // bfmmla z19.s, z2.h, z6.h\n"
       ".inst 0x6466e49b  // bfmmla z27.s, z4.h, z6.h\n"
@@ -1653,21 +1653,21 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       ".inst 0x6467e49f  // bfmmla z31.s, z4.h, z7.h\n"
       "76:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 71b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x24, x28, x19, LSL #2\n"
-      "add x23, x24, x19, LSL #2\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x25, x9, x20, LSL #2\n"
+      "add x24, x25, x20, LSL #2\n"
       "uzp1 z6.d, z8.d, z12.d\n"
-      "add x22, x23, x19, LSL #2\n"
-      "add x21, x22, x19, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
       "uzp1 z12.d, z9.d, z13.d\n"
       "uzp2 z9.d, z9.d, z13.d\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z10.d, z10.d, z14.d\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
@@ -1688,10 +1688,10 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "uzp1 z30.d, z27.d, z31.d\n"
       "uzp2 z27.d, z27.d, z31.d\n"
       "tbz %x[flags], #1, 77f\n"
-      "add x19, %x[args_ptr], %[offset_max]\n"
-      "ld1rw { z1.s }, p5/Z, [x19]\n"
-      "add x19, %x[args_ptr], %[offset_min]\n"
-      "ld1rw { z0.s }, p5/Z, [x19]\n"
+      "add x20, %x[args_ptr], %[offset_max]\n"
+      "ld1rw { z1.s }, p5/Z, [x20]\n"
+      "add x20, %x[args_ptr], %[offset_min]\n"
+      "ld1rw { z0.s }, p5/Z, [x20]\n"
       "fmin z6.s, p5/M, z6.s, z1.s\n"
       "fmin z12.s, p5/M, z12.s, z1.s\n"
       "fmin z13.s, p5/M, z13.s, z1.s\n"
@@ -1741,51 +1741,51 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL (
       "fmax z26.s, p5/M, z26.s, z0.s\n"
       "fmax z27.s, p5/M, z27.s, z0.s\n"
       "77:"  // Height 6: No activation
-      "st1w { z6.s }, p4, [x28]\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x24]\n"
-      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
-      "st1w { z15.s }, p4, [x23]\n"
-      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z23.s }, p4, [x21]\n"
-      "st1w { z28.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z29.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z30.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z6.s }, p4, [x9]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x25]\n"
+      "st1w { z9.s }, p3, [x25, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x25, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x25, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x24]\n"
+      "st1w { z20.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z23.s }, p4, [x22]\n"
+      "st1w { z28.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z29.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z30.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "78:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0x18\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x18\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp
index 3031f5abf5..562b2759aa 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -85,165 +85,165 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z15.b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x23, #0x10\n"
       "sdot z17.s, z5.b, z0.b[0]\n"
+      "sdot z18.s, z6.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z19.s, z7.b, z0.b[0]\n"
       "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "sdot z18.s, z6.b, z0.b[0]\n"
+      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "sdot z19.s, z7.b, z0.b[0]\n"
-      "sdot z16.s, z8.b, z0.b[1]\n"
+      "sdot z17.s, z9.b, z0.b[1]\n"
+      "sdot z18.s, z10.b, z0.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      "sdot z17.s, z9.b, z0.b[1]\n"
+      "sdot z19.s, z4.b, z0.b[1]\n"
       "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
-      "sdot z18.s, z10.b, z0.b[1]\n"
       "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
-      "sdot z19.s, z4.b, z0.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "sdot z16.s, z5.b, z0.b[2]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z19.s, z8.b, z0.b[2]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z16.s, z9.b, z0.b[3]\n"
       "sdot z17.s, z10.b, z0.b[3]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "sdot z18.s, z4.b, z0.b[3]\n"
       "sdot z19.s, z5.b, z0.b[3]\n"
+      "add x24, x24, #0x10\n"
       "tbnz %x[flags], #31, 8f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      "subs x25, x25, #0x4\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
-      "sdot z16.s, z4.b, z0.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z16.s, z4.b, z0.b[0]\n"
       "sdot z17.s, z5.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
       "sdot z19.s, z7.b, z0.b[0]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z17.s, z9.b, z0.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z17.s, z9.b, z0.b[1]\n"
       "sdot z18.s, z10.b, z0.b[1]\n"
-      "addvl x28, x28, #4\n"
       "sdot z19.s, z4.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z17.s, z6.b, z0.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z17.s, z6.b, z0.b[2]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
-      "addvl x28, x28, #4\n"
       "sdot z19.s, z8.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z16.s, z9.b, z0.b[3]\n"
       "sdot z17.s, z10.b, z0.b[3]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "sdot z18.s, z4.b, z0.b[3]\n"
       "sdot z19.s, z5.b, z0.b[3]\n"
+      "addvl x28, x28, #4\n"
       "10:"  // Height 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "11:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "tbnz %x[flags], #31, 12f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z1.s }, p2/Z, [x19]\n"
-      "neg z1.s, p2/M, z1.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z1.s }, p2/Z, [x20]\n"
       "saddv d11, p0, z11.s\n"
       "mov z11.s, z11.s[0]\n"
+      "neg z1.s, p2/M, z1.s\n"
       "mul z11.s, p2/M, z11.s, z1.s\n"
       "12:"  // Height 1: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z16.s, z16.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
       ".inst 0x04a47631  // sqrdmulh z17.s, z17.s, z4.s\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x04a47652  // sqrdmulh z18.s, z18.s, z4.s\n"
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 13f\n"
@@ -254,26 +254,26 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "13:"  // Height 1: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
       "add z19.s, z19.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
@@ -281,31 +281,31 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "addvl x26, x26, #1\n"
+      "st1b { z16.b }, p1, [x27]\n"
+      "addvl x27, x27, #1\n"
       "14:"  // Height 1: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 2b\n"
       "b 58f\n"
       "15:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z15.b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -313,68 +313,68 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "18:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "20:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 23f\n"
       "21:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
       "sdot z21.s, z5.b, z1.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "sdot z22.s, z6.b, z1.b[0]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "sdot z19.s, z7.b, z0.b[0]\n"
+      "sdot z23.s, z7.b, z1.b[0]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      "sdot z23.s, z7.b, z1.b[0]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z16.s, z8.b, z0.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
       "sdot z20.s, z8.b, z1.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z17.s, z9.b, z0.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "sdot z21.s, z9.b, z1.b[1]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "sdot z18.s, z10.b, z0.b[1]\n"
       "sdot z22.s, z10.b, z1.b[1]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z19.s, z4.b, z0.b[1]\n"
       "sdot z23.s, z4.b, z1.b[1]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z16.s, z5.b, z0.b[2]\n"
       "sdot z20.s, z5.b, z1.b[2]\n"
       "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
       "sdot z21.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
@@ -393,69 +393,69 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z12.s, z1.b, z15.b\n"
       "22:"  // Height 2: Multiply loop: unique 3: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 21b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
+      "sdot z20.s, z4.b, z1.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "sdot z20.s, z4.b, z1.b[0]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
       "sdot z21.s, z5.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
       "sdot z22.s, z6.b, z1.b[0]\n"
+      "addvl x28, x28, #4\n"
       "sdot z19.s, z7.b, z0.b[0]\n"
       "sdot z23.s, z7.b, z1.b[0]\n"
       "ble 24f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z20.s, z8.b, z1.b[1]\n"
       "sdot z17.s, z9.b, z0.b[1]\n"
-      "addvl x28, x28, #4\n"
       "sdot z21.s, z9.b, z1.b[1]\n"
       "sdot z18.s, z10.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "sdot z22.s, z10.b, z1.b[1]\n"
       "sdot z19.s, z4.b, z0.b[1]\n"
       "sdot z23.s, z4.b, z1.b[1]\n"
       "ble 24f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "sdot z20.s, z5.b, z1.b[2]\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
-      "addvl x28, x28, #4\n"
       "sdot z21.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "sdot z22.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z8.b, z0.b[2]\n"
       "sdot z23.s, z8.b, z1.b[2]\n"
       "ble 24f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "sdot z16.s, z9.b, z0.b[3]\n"
       "sdot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
       "sdot z17.s, z10.b, z0.b[3]\n"
-      "addvl x28, x28, #4\n"
       "sdot z21.s, z10.b, z1.b[3]\n"
       "sdot z18.s, z4.b, z0.b[3]\n"
       "sdot z22.s, z4.b, z1.b[3]\n"
+      "addvl x28, x28, #4\n"
       "sdot z19.s, z5.b, z0.b[3]\n"
       "sdot z23.s, z5.b, z1.b[3]\n"
       "24:"  // Height 2: Multiply loop: multiply skip
@@ -463,49 +463,49 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z12.s, z1.b, z15.b\n"
       "25:"  // Height 2: Multiply loop: unique 4: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 18b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
       "tbnz %x[flags], #31, 26f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z2.s }, p2/Z, [x19]\n"
-      "neg z2.s, p2/M, z2.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z2.s }, p2/Z, [x20]\n"
       "saddv d11, p0, z11.s\n"
-      "saddv d12, p0, z12.s\n"
       "mov z11.s, z11.s[0]\n"
+      "saddv d12, p0, z12.s\n"
+      "neg z2.s, p2/M, z2.s\n"
       "mov z12.s, z12.s[0]\n"
       "mul z11.s, p2/M, z11.s, z2.s\n"
       "mul z12.s, p2/M, z12.s, z2.s\n"
       "26:"  // Height 2: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
+      "addvl x10, x10, #4\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       "add z20.s, z20.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z21.s, z21.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z22.s, z22.s, z2.s\n"
       "add z23.s, z23.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -518,97 +518,97 @@ void sve_hybrid_s8qa_dot_4x4VL (
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
       "tbz %x[flags], #5, 27f\n"
       "and z4.d, z16.d, z0.d\n"
-      "and z5.d, z17.d, z0.d\n"
-      "and z6.d, z18.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
-      "sqadd z17.s, z17.s, z5.s\n"
-      "sqadd z18.s, z18.s, z6.s\n"
+      "and z5.d, z17.d, z0.d\n"
+      "and z6.d, z18.d, z0.d\n"
       "and z7.d, z19.d, z0.d\n"
       "and z8.d, z20.d, z0.d\n"
       "and z9.d, z21.d, z0.d\n"
+      "and z10.d, z22.d, z0.d\n"
+      "and z4.d, z23.d, z0.d\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z17.s, z17.s, z5.s\n"
+      "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "sqadd z20.s, z20.s, z8.s\n"
       "sqadd z21.s, z21.s, z9.s\n"
-      "and z10.d, z22.d, z0.d\n"
-      "and z4.d, z23.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
       "sqadd z22.s, z22.s, z10.s\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "27:"  // Height 2: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
+      "st1b { z20.b }, p1, [x23]\n"
+      "addvl x27, x27, #1\n"
       "28:"  // Height 2: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 16b\n"
       "b 58f\n"
       "29:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "mov z15.b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -620,83 +620,83 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "34:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 37f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
       "sdot z24.s, z4.b, z2.b[0]\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
       "sdot z21.s, z5.b, z1.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "sdot z25.s, z5.b, z2.b[0]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "sdot z22.s, z6.b, z1.b[0]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "sdot z26.s, z6.b, z2.b[0]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z19.s, z7.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       "sdot z23.s, z7.b, z1.b[0]\n"
       "sdot z27.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
       "sdot z16.s, z8.b, z0.b[1]\n"
       "sdot z20.s, z8.b, z1.b[1]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       "sdot z24.s, z8.b, z2.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "sdot z17.s, z9.b, z0.b[1]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       "sdot z21.s, z9.b, z1.b[1]\n"
       "sdot z25.s, z9.b, z2.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z18.s, z10.b, z0.b[1]\n"
       "sdot z22.s, z10.b, z1.b[1]\n"
       "sdot z26.s, z10.b, z2.b[1]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "sdot z19.s, z4.b, z0.b[1]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "sdot z23.s, z4.b, z1.b[1]\n"
       "sdot z27.s, z4.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z16.s, z5.b, z0.b[2]\n"
       "sdot z20.s, z5.b, z1.b[2]\n"
       "sdot z24.s, z5.b, z2.b[2]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "sdot z21.s, z6.b, z1.b[2]\n"
       "sdot z25.s, z6.b, z2.b[2]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
@@ -722,26 +722,26 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z12.s, z1.b, z15.b\n"
       "sdot z13.s, z2.b, z15.b\n"
       "36:"  // Height 3: Multiply loop: unique 5: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 35b\n"
       "37:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z24.s, z4.b, z2.b[0]\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
       "sdot z21.s, z5.b, z1.b[0]\n"
       "sdot z25.s, z5.b, z2.b[0]\n"
+      "addvl x28, x28, #4\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
       "sdot z22.s, z6.b, z1.b[0]\n"
       "sdot z26.s, z6.b, z2.b[0]\n"
@@ -750,16 +750,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z27.s, z7.b, z2.b[0]\n"
       "ble 38f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z8.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z20.s, z8.b, z1.b[1]\n"
+      "sdot z24.s, z8.b, z2.b[1]\n"
       "sdot z17.s, z9.b, z0.b[1]\n"
       "sdot z21.s, z9.b, z1.b[1]\n"
+      "addvl x28, x28, #4\n"
       "sdot z25.s, z9.b, z2.b[1]\n"
       "sdot z18.s, z10.b, z0.b[1]\n"
       "sdot z22.s, z10.b, z1.b[1]\n"
@@ -769,16 +769,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z27.s, z4.b, z2.b[1]\n"
       "ble 38f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z5.b, z2.b[2]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z20.s, z5.b, z1.b[2]\n"
+      "sdot z24.s, z5.b, z2.b[2]\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
       "sdot z21.s, z6.b, z1.b[2]\n"
+      "addvl x28, x28, #4\n"
       "sdot z25.s, z6.b, z2.b[2]\n"
       "sdot z18.s, z7.b, z0.b[2]\n"
       "sdot z22.s, z7.b, z1.b[2]\n"
@@ -788,16 +788,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z27.s, z8.b, z2.b[2]\n"
       "ble 38f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "sdot z16.s, z9.b, z0.b[3]\n"
       "sdot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z9.b, z2.b[3]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z24.s, z9.b, z2.b[3]\n"
       "sdot z17.s, z10.b, z0.b[3]\n"
       "sdot z21.s, z10.b, z1.b[3]\n"
       "sdot z25.s, z10.b, z2.b[3]\n"
+      "addvl x28, x28, #4\n"
       "sdot z18.s, z4.b, z0.b[3]\n"
       "sdot z22.s, z4.b, z1.b[3]\n"
       "sdot z26.s, z4.b, z2.b[3]\n"
@@ -810,48 +810,48 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z12.s, z1.b, z15.b\n"
       "sdot z13.s, z2.b, z15.b\n"
       "39:"  // Height 3: Multiply loop: unique 6: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
-      "add x21, x22, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
+      "add x22, x23, x20\n"
       "tbnz %x[flags], #31, 40f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z3.s }, p2/Z, [x19]\n"
-      "neg z3.s, p2/M, z3.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z3.s }, p2/Z, [x20]\n"
       "saddv d11, p0, z11.s\n"
+      "mov z11.s, z11.s[0]\n"
       "saddv d12, p0, z12.s\n"
       "saddv d13, p0, z13.s\n"
-      "mov z11.s, z11.s[0]\n"
       "mov z12.s, z12.s[0]\n"
       "mov z13.s, z13.s[0]\n"
+      "neg z3.s, p2/M, z3.s\n"
       "mul z11.s, p2/M, z11.s, z3.s\n"
       "mul z12.s, p2/M, z12.s, z3.s\n"
       "mul z13.s, p2/M, z13.s, z3.s\n"
       "40:"  // Height 3: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z16.s, z16.s, z0.s\n"
@@ -863,8 +863,8 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "add z22.s, z22.s, z2.s\n"
       "add z23.s, z23.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -883,131 +883,131 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z0.d\n"
       "and z6.d, z18.d, z0.d\n"
+      "and z7.d, z19.d, z0.d\n"
+      "and z8.d, z20.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "and z7.d, z19.d, z0.d\n"
-      "and z8.d, z20.d, z0.d\n"
-      "and z9.d, z21.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "sqadd z20.s, z20.s, z8.s\n"
-      "sqadd z21.s, z21.s, z9.s\n"
+      "and z9.d, z21.d, z0.d\n"
       "and z10.d, z22.d, z0.d\n"
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z22.s, z22.s, z10.s\n"
-      "sqadd z23.s, z23.s, z4.s\n"
-      "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
+      "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
+      "sqadd z21.s, z21.s, z9.s\n"
+      "sqadd z22.s, z22.s, z10.s\n"
+      "sqadd z23.s, z23.s, z4.s\n"
+      "sqadd z24.s, z24.s, z5.s\n"
       "sqadd z25.s, z25.s, z6.s\n"
       "sqadd z26.s, z26.s, z7.s\n"
       "sqadd z27.s, z27.s, z8.s\n"
       "41:"  // Height 3: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x23]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x21]\n"
+      "st1b { z24.b }, p1, [x22]\n"
+      "addvl x27, x27, #1\n"
       "42:"  // Height 3: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 30b\n"
       "b 58f\n"
       "43:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z14.s, #0x0\n"
-      "mov x19, #0x4\n"
       "mov z15.b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -1023,66 +1023,66 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "46:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "48:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 51f\n"
       "49:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "ld1rqb { z3.b }, p0/Z, [x21]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x20]\n"
-      "add x21, x21, #0x10\n"
-      "sdot z24.s, z4.b, z2.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x20, x20, #0x10\n"
-      "sdot z21.s, z5.b, z1.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "sdot z25.s, z5.b, z2.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "sdot z24.s, z4.b, z2.b[0]\n"
       "sdot z28.s, z4.b, z3.b[0]\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
+      "sdot z21.s, z5.b, z1.b[0]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "sdot z25.s, z5.b, z2.b[0]\n"
       "sdot z29.s, z5.b, z3.b[0]\n"
       "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "sdot z18.s, z6.b, z0.b[0]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "sdot z18.s, z6.b, z0.b[0]\n"
       "sdot z22.s, z6.b, z1.b[0]\n"
+      "addvl x28, x28, #16\n"
       "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z26.s, z6.b, z2.b[0]\n"
       "sdot z30.s, z6.b, z3.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "add x21, x21, #0x10\n"
       "sdot z19.s, z7.b, z0.b[0]\n"
       "sdot z23.s, z7.b, z1.b[0]\n"
       "sdot z27.s, z7.b, z2.b[0]\n"
@@ -1147,27 +1147,27 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z13.s, z2.b, z15.b\n"
       "sdot z14.s, z3.b, z15.b\n"
       "50:"  // Height 4: Multiply loop: unique 7: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 49b\n"
       "51:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "ld1rqb { z3.b }, p0/Z, [x21]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "sdot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "sdot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "ld1rqb { z3.b }, p0/Z, [x20]\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z21.s, z5.b, z1.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "sdot z24.s, z4.b, z2.b[0]\n"
       "sdot z28.s, z4.b, z3.b[0]\n"
+      "sdot z17.s, z5.b, z0.b[0]\n"
+      "sdot z21.s, z5.b, z1.b[0]\n"
+      "addvl x28, x28, #4\n"
       "sdot z25.s, z5.b, z2.b[0]\n"
       "sdot z29.s, z5.b, z3.b[0]\n"
       "sdot z18.s, z6.b, z0.b[0]\n"
@@ -1180,16 +1180,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z31.s, z7.b, z3.b[0]\n"
       "ble 52f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z8.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z20.s, z8.b, z1.b[1]\n"
+      "sdot z24.s, z8.b, z2.b[1]\n"
       "sdot z28.s, z8.b, z3.b[1]\n"
       "sdot z17.s, z9.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "sdot z21.s, z9.b, z1.b[1]\n"
       "sdot z25.s, z9.b, z2.b[1]\n"
       "sdot z29.s, z9.b, z3.b[1]\n"
@@ -1203,16 +1203,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z31.s, z4.b, z3.b[1]\n"
       "ble 52f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "sdot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "sdot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z5.b, z2.b[2]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z20.s, z5.b, z1.b[2]\n"
+      "sdot z24.s, z5.b, z2.b[2]\n"
       "sdot z28.s, z5.b, z3.b[2]\n"
       "sdot z17.s, z6.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "sdot z21.s, z6.b, z1.b[2]\n"
       "sdot z25.s, z6.b, z2.b[2]\n"
       "sdot z29.s, z6.b, z3.b[2]\n"
@@ -1226,16 +1226,16 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z31.s, z8.b, z3.b[2]\n"
       "ble 52f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "sdot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "sdot z16.s, z9.b, z0.b[3]\n"
       "sdot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "sdot z24.s, z9.b, z2.b[3]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "sdot z24.s, z9.b, z2.b[3]\n"
       "sdot z28.s, z9.b, z3.b[3]\n"
       "sdot z17.s, z10.b, z0.b[3]\n"
       "sdot z21.s, z10.b, z1.b[3]\n"
+      "addvl x28, x28, #4\n"
       "sdot z25.s, z10.b, z2.b[3]\n"
       "sdot z29.s, z10.b, z3.b[3]\n"
       "sdot z18.s, z4.b, z0.b[3]\n"
@@ -1253,27 +1253,27 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "sdot z13.s, z2.b, z15.b\n"
       "sdot z14.s, z3.b, z15.b\n"
       "53:"  // Height 4: Multiply loop: unique 8: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 46b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "tbnz %x[flags], #31, 54f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "neg z4.s, p2/M, z4.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       "saddv d11, p0, z11.s\n"
+      "mov z11.s, z11.s[0]\n"
       "saddv d12, p0, z12.s\n"
       "saddv d13, p0, z13.s\n"
-      "saddv d14, p0, z14.s\n"
-      "mov z11.s, z11.s[0]\n"
       "mov z12.s, z12.s[0]\n"
       "mov z13.s, z13.s[0]\n"
+      "saddv d14, p0, z14.s\n"
+      "neg z4.s, p2/M, z4.s\n"
       "mov z14.s, z14.s[0]\n"
       "mul z11.s, p2/M, z11.s, z4.s\n"
       "mul z12.s, p2/M, z12.s, z4.s\n"
@@ -1281,24 +1281,24 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "mul z14.s, p2/M, z14.s, z4.s\n"
       "54:"  // Height 4: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z28.s, z28.s, z14.s\n"
@@ -1318,8 +1318,8 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       "add z28.s, z28.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z29.s, z29.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z30.s, z30.s, z2.s\n"
       "add z31.s, z31.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -1341,160 +1341,160 @@ void sve_hybrid_s8qa_dot_4x4VL (
       "tbz %x[flags], #5, 55f\n"
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z0.d\n"
-      "and z6.d, z18.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
-      "sqadd z18.s, z18.s, z6.s\n"
+      "and z6.d, z18.d, z0.d\n"
       "and z7.d, z19.d, z0.d\n"
       "and z8.d, z20.d, z0.d\n"
       "and z9.d, z21.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
-      "sqadd z19.s, z19.s, z7.s\n"
-      "sqadd z20.s, z20.s, z8.s\n"
-      "sqadd z21.s, z21.s, z9.s\n"
       "and z10.d, z22.d, z0.d\n"
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
+      "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z18.s, z18.s, z6.s\n"
+      "sqadd z19.s, z19.s, z7.s\n"
+      "sqadd z20.s, z20.s, z8.s\n"
+      "sqadd z21.s, z21.s, z9.s\n"
       "sqadd z22.s, z22.s, z10.s\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
-      "asr z6.s, z6.s, #0x1f\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "sqadd z25.s, z25.s, z6.s\n"
-      "sqadd z26.s, z26.s, z7.s\n"
-      "sqadd z27.s, z27.s, z8.s\n"
       "and z9.d, z28.d, z0.d\n"
       "and z10.d, z29.d, z0.d\n"
       "and z4.d, z30.d, z0.d\n"
+      "and z5.d, z31.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z25.s, z25.s, z6.s\n"
+      "sqadd z26.s, z26.s, z7.s\n"
+      "sqadd z27.s, z27.s, z8.s\n"
       "sqadd z28.s, z28.s, z9.s\n"
       "sqadd z29.s, z29.s, z10.s\n"
       "sqadd z30.s, z30.s, z4.s\n"
-      "and z5.d, z31.d, z0.d\n"
-      "asr z5.s, z5.s, #0x1f\n"
       "sqadd z31.s, z31.s, z5.s\n"
       "55:"  // Height 4: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z31.s, z31.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z31.s, p2/M, z31.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
-      "add z28.s, z28.s, z4.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x23]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
-      "add z29.s, z29.s, z4.s\n"
-      "uzp1 z25.h, z26.h, z27.h\n"
       "smax z28.s, p2/M, z28.s, z5.s\n"
-      "add z30.s, z30.s, z4.s\n"
+      "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x21]\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
-      ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "uzp1 z28.h, z28.h, z29.h\n"
-      "smin z31.s, p2/M, z31.s, z6.s\n"
+      "st1b { z24.b }, p1, [x22]\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "uzp1 z29.h, z30.h, z31.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p1, [x20]\n"
+      "st1b { z28.b }, p1, [x21]\n"
+      "addvl x27, x27, #1\n"
       "56:"  // Height 4: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 44b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 58f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "57:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "58:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp
index 04f80982e8..626a06b26b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -85,18 +85,18 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z15.b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -104,119 +104,119 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x23, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
       ".inst 0x45069837  // smmla z23.s, z1.b, z6.b\n"
+      "add x24, x24, #0x10\n"
       "tbnz %x[flags], #31, 8f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z11.s, z1.b, z15.b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
+      "addvl x28, x28, #8\n"
       "ble 10f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
+      ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
-      ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
       ".inst 0x45069837  // smmla z23.s, z1.b, z6.b\n"
+      "addvl x28, x28, #8\n"
       "10:"  // Height 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z11.s, z1.b, z15.b\n"
       "11:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "uzp1 z16.d, z16.d, z20.d\n"
       "uzp1 z17.d, z17.d, z21.d\n"
@@ -224,33 +224,33 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "uzp1 z19.d, z19.d, z23.d\n"
       "mov z23.d, z16.d\n"
       "tbnz %x[flags], #31, 12f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z1.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z1.s }, p2/Z, [x22]\n"
-      "mov z11.s, z11.s[0]\n"
       "neg z1.s, p2/M, z1.s\n"
+      "mov z11.s, z11.s[0]\n"
       "mul z11.s, p2/M, z11.s, z1.s\n"
       "12:"  // Height 1: skip row sum fixup
       "add z23.s, z23.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z23.s, z23.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
       ".inst 0x04a47631  // sqrdmulh z17.s, z17.s, z4.s\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x04a47652  // sqrdmulh z18.s, z18.s, z4.s\n"
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 13f\n"
@@ -261,26 +261,26 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "13:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
       "add z19.s, z19.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
@@ -288,31 +288,31 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z23.h, z23.h, z17.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z23.b, z23.b, z17.b\n"
-      "st1b { z23.b }, p1, [x26]\n"
-      "addvl x26, x26, #1\n"
+      "st1b { z23.b }, p1, [x27]\n"
+      "addvl x27, x27, #1\n"
       "14:"  // Height 1: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 2b\n"
       "b 58f\n"
       "15:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z15.b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -320,130 +320,130 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "18:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "20:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 23f\n"
       "21:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
       ".inst 0x45069837  // smmla z23.s, z1.b, z6.b\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "tbnz %x[flags], #31, 22f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z11.s, z1.b, z15.b\n"
       "22:"  // Height 2: Multiply loop: unique 3: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 21b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
+      "addvl x28, x28, #8\n"
       "ble 24f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
+      ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
-      ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
       ".inst 0x45069837  // smmla z23.s, z1.b, z6.b\n"
+      "addvl x28, x28, #8\n"
       "24:"  // Height 2: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 25f\n"
       "sdot z11.s, z0.b, z15.b\n"
       "sdot z11.s, z1.b, z15.b\n"
       "25:"  // Height 2: Multiply loop: unique 4: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 18b\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
+      "add x22, x27, x20\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
       "uzp1 z21.d, z18.d, z22.d\n"
@@ -452,39 +452,39 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "uzp2 z19.d, z19.d, z23.d\n"
       "mov z23.d, z7.d\n"
       "tbnz %x[flags], #31, 26f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z2.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z2.s }, p2/Z, [x22]\n"
+      "neg z2.s, p2/M, z2.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z2.s, p2/M, z2.s\n"
       "mul z11.s, p2/M, z11.s, z2.s\n"
       "mul z12.s, p2/M, z12.s, z2.s\n"
       "26:"  // Height 2: skip row sum fixup
       "add z23.s, z23.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z23.s, z23.s, z0.s\n"
       "add z20.s, z20.s, z1.s\n"
+      "addvl x10, x10, #4\n"
       "add z21.s, z21.s, z2.s\n"
       "add z22.s, z22.s, z3.s\n"
       "add z16.s, z16.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
@@ -497,97 +497,97 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 27f\n"
       "and z4.d, z23.d, z0.d\n"
-      "and z5.d, z20.d, z0.d\n"
-      "and z6.d, z21.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
-      "sqadd z20.s, z20.s, z5.s\n"
-      "sqadd z21.s, z21.s, z6.s\n"
+      "and z5.d, z20.d, z0.d\n"
+      "and z6.d, z21.d, z0.d\n"
       "and z7.d, z22.d, z0.d\n"
       "and z8.d, z16.d, z0.d\n"
       "and z9.d, z17.d, z0.d\n"
+      "and z10.d, z18.d, z0.d\n"
+      "and z4.d, z19.d, z0.d\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z20.s, z20.s, z5.s\n"
+      "sqadd z21.s, z21.s, z6.s\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "sqadd z16.s, z16.s, z8.s\n"
       "sqadd z17.s, z17.s, z9.s\n"
-      "and z10.d, z18.d, z0.d\n"
-      "and z4.d, z19.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
       "sqadd z18.s, z18.s, z10.s\n"
       "sqadd z19.s, z19.s, z4.s\n"
       "27:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z23.h, z23.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z23.b, z23.b, z20.b\n"
-      "st1b { z23.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z23.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
+      "st1b { z16.b }, p1, [x22]\n"
+      "addvl x27, x27, #1\n"
       "28:"  // Height 2: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 16b\n"
       "b 58f\n"
       "29:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "mov z15.b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -603,74 +603,74 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "34:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 37f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
+      ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
-      ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
       ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
       ".inst 0x4508985d  // smmla z29.s, z2.b, z8.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
       ".inst 0x4509985a  // smmla z26.s, z2.b, z9.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
       ".inst 0x450a985e  // smmla z30.s, z2.b, z10.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
       ".inst 0x4504985b  // smmla z27.s, z2.b, z4.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
       ".inst 0x4505985f  // smmla z31.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
       ".inst 0x45069878  // smmla z24.s, z3.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
@@ -694,36 +694,36 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "sdot z11.s, z1.b, z15.b\n"
       "sdot z13.s, z3.b, z15.b\n"
       "36:"  // Height 3: Multiply loop: unique 5: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 35b\n"
       "37:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
-      ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
+      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
-      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
       ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
       ".inst 0x4508985d  // smmla z29.s, z2.b, z8.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
       ".inst 0x4509985a  // smmla z26.s, z2.b, z9.b\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
@@ -735,23 +735,23 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "ble 38f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45069878  // smmla z24.s, z3.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4507987c  // smmla z28.s, z3.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45089879  // smmla z25.s, z3.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
       ".inst 0x4509987d  // smmla z29.s, z3.b, z9.b\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
       ".inst 0x450a987a  // smmla z26.s, z3.b, z10.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
       ".inst 0x4504987e  // smmla z30.s, z3.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
@@ -765,17 +765,17 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "sdot z11.s, z1.b, z15.b\n"
       "sdot z13.s, z3.b, z15.b\n"
       "39:"  // Height 3: Multiply loop: unique 6: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 32b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "add x20, x21, x19\n"
+      "add x21, x22, x20\n"
       "uzp1 z21.d, z18.d, z22.d\n"
       "uzp2 z18.d, z18.d, z22.d\n"
       "uzp1 z22.d, z19.d, z23.d\n"
@@ -786,37 +786,37 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "uzp1 z27.d, z27.d, z31.d\n"
       "mov z31.d, z7.d\n"
       "tbnz %x[flags], #31, 40f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z3.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z3.s }, p2/Z, [x22]\n"
       ".inst 0x4491a9ad  // addp z13.s, p2/m, z13.s, z13.s\n"
+      "neg z3.s, p2/M, z3.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z3.s, p2/M, z3.s\n"
-      "mov z13.s, z13.s[0]\n"
       "mul z11.s, p2/M, z11.s, z3.s\n"
+      "mov z13.s, z13.s[0]\n"
       "mul z12.s, p2/M, z12.s, z3.s\n"
       "mul z13.s, p2/M, z13.s, z3.s\n"
       "40:"  // Height 3: skip row sum fixup
       "add z31.s, z31.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z31.s, z31.s, z0.s\n"
@@ -828,8 +828,8 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -848,131 +848,131 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z20.d, z0.d\n"
       "and z6.d, z21.d, z0.d\n"
+      "and z7.d, z22.d, z0.d\n"
+      "and z8.d, z16.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
       "sqadd z21.s, z21.s, z6.s\n"
-      "and z7.d, z22.d, z0.d\n"
-      "and z8.d, z16.d, z0.d\n"
-      "and z9.d, z17.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "sqadd z16.s, z16.s, z8.s\n"
-      "sqadd z17.s, z17.s, z9.s\n"
+      "and z9.d, z17.d, z0.d\n"
       "and z10.d, z18.d, z0.d\n"
       "and z4.d, z19.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z18.s, z18.s, z10.s\n"
-      "sqadd z19.s, z19.s, z4.s\n"
-      "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
+      "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
+      "sqadd z17.s, z17.s, z9.s\n"
+      "sqadd z18.s, z18.s, z10.s\n"
+      "sqadd z19.s, z19.s, z4.s\n"
+      "sqadd z24.s, z24.s, z5.s\n"
       "sqadd z25.s, z25.s, z6.s\n"
       "sqadd z26.s, z26.s, z7.s\n"
       "sqadd z27.s, z27.s, z8.s\n"
       "41:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z31.h, z31.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z31.b, z31.b, z20.b\n"
-      "st1b { z31.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z31.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x20]\n"
+      "st1b { z24.b }, p1, [x21]\n"
+      "addvl x27, x27, #1\n"
       "42:"  // Height 3: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 30b\n"
       "b 58f\n"
       "43:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z14.s, #0x0\n"
-      "mov x19, #0x4\n"
       "mov z15.b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -988,85 +988,85 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "46:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "48:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 51f\n"
       "49:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
+      "ld1rqb { z4.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x20]\n"
-      "add x21, x21, #0x10\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
+      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
+      ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
+      ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      ".inst 0x4508985d  // smmla z29.s, z2.b, z8.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
-      ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
-      ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
-      ".inst 0x4508985d  // smmla z29.s, z2.b, z8.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
       ".inst 0x4509985a  // smmla z26.s, z2.b, z9.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x450a985e  // smmla z30.s, z2.b, z10.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x45049813  // smmla z19.s, z0.b, z4.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x4504985b  // smmla z27.s, z2.b, z4.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45059817  // smmla z23.s, z0.b, z5.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4505985f  // smmla z31.s, z2.b, z5.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45069878  // smmla z24.s, z3.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4507987c  // smmla z28.s, z3.b, z7.b\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x45089879  // smmla z25.s, z3.b, z8.b\n"
       ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
       ".inst 0x4509987d  // smmla z29.s, z3.b, z9.b\n"
@@ -1084,38 +1084,38 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "sdot z11.s, z1.b, z15.b\n"
       "sdot z13.s, z3.b, z15.b\n"
       "50:"  // Height 4: Multiply loop: unique 7: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 49b\n"
       "51:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
+      "ld1rqb { z4.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x20]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45059810  // smmla z16.s, z0.b, z5.b\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "subs x25, x25, #0x8\n"
       ".inst 0x45069814  // smmla z20.s, z0.b, z6.b\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
+      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       ".inst 0x45079811  // smmla z17.s, z0.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
-      ".inst 0x4506985c  // smmla z28.s, z2.b, z6.b\n"
       ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
       ".inst 0x45089815  // smmla z21.s, z0.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x4508985d  // smmla z29.s, z2.b, z8.b\n"
       ".inst 0x45099812  // smmla z18.s, z0.b, z9.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x4509985a  // smmla z26.s, z2.b, z9.b\n"
       ".inst 0x450a9816  // smmla z22.s, z0.b, z10.b\n"
       ".inst 0x450a985e  // smmla z30.s, z2.b, z10.b\n"
@@ -1126,23 +1126,23 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "ble 52f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45069878  // smmla z24.s, z3.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45079834  // smmla z20.s, z1.b, z7.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4507987c  // smmla z28.s, z3.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45089831  // smmla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45089879  // smmla z25.s, z3.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45099835  // smmla z21.s, z1.b, z9.b\n"
       ".inst 0x4509987d  // smmla z29.s, z3.b, z9.b\n"
       ".inst 0x450a9832  // smmla z18.s, z1.b, z10.b\n"
       ".inst 0x450a987a  // smmla z26.s, z3.b, z10.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45049836  // smmla z22.s, z1.b, z4.b\n"
       ".inst 0x4504987e  // smmla z30.s, z3.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
@@ -1156,19 +1156,19 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "sdot z11.s, z1.b, z15.b\n"
       "sdot z13.s, z3.b, z15.b\n"
       "53:"  // Height 4: Multiply loop: unique 8: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 46b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
       "uzp1 z20.d, z17.d, z21.d\n"
+      "add x20, x21, x20\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "add x20, x21, x19\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "add x19, x20, x19\n"
       "uzp2 z18.d, z18.d, z22.d\n"
       "uzp1 z22.d, z19.d, z23.d\n"
       "uzp2 z19.d, z19.d, z23.d\n"
@@ -1182,39 +1182,39 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "uzp2 z27.d, z27.d, z31.d\n"
       "mov z31.d, z7.d\n"
       "tbnz %x[flags], #31, 54f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       ".inst 0x4491a9ad  // addp z13.s, p2/m, z13.s, z13.s\n"
+      "neg z4.s, p2/M, z4.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z4.s, p2/M, z4.s\n"
+      "mul z11.s, p2/M, z11.s, z4.s\n"
       "mov z14.s, z13.s[3]\n"
       "mov z13.s, z13.s[0]\n"
-      "mul z11.s, p2/M, z11.s, z4.s\n"
       "mul z12.s, p2/M, z12.s, z4.s\n"
       "mul z13.s, p2/M, z13.s, z4.s\n"
       "mul z14.s, p2/M, z14.s, z4.s\n"
       "54:"  // Height 4: skip row sum fixup
       "add z31.s, z31.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z23.s, z23.s, z13.s\n"
       "add z28.s, z28.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z29.s, z29.s, z13.s\n"
       "add z30.s, z30.s, z13.s\n"
       "add z24.s, z24.s, z14.s\n"
@@ -1234,8 +1234,8 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "add z29.s, z29.s, z2.s\n"
       "add z30.s, z30.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -1257,160 +1257,160 @@ void sve_hybrid_s8qa_mmla_4x4VL (
       "tbz %x[flags], #5, 55f\n"
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z20.d, z0.d\n"
-      "and z6.d, z21.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
-      "sqadd z21.s, z21.s, z6.s\n"
+      "and z6.d, z21.d, z0.d\n"
       "and z7.d, z22.d, z0.d\n"
       "and z8.d, z16.d, z0.d\n"
       "and z9.d, z17.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
-      "sqadd z22.s, z22.s, z7.s\n"
-      "sqadd z16.s, z16.s, z8.s\n"
-      "sqadd z17.s, z17.s, z9.s\n"
       "and z10.d, z18.d, z0.d\n"
       "and z4.d, z19.d, z0.d\n"
       "and z5.d, z23.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
+      "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z21.s, z21.s, z6.s\n"
+      "sqadd z22.s, z22.s, z7.s\n"
+      "sqadd z16.s, z16.s, z8.s\n"
+      "sqadd z17.s, z17.s, z9.s\n"
       "sqadd z18.s, z18.s, z10.s\n"
       "sqadd z19.s, z19.s, z4.s\n"
       "sqadd z23.s, z23.s, z5.s\n"
       "and z6.d, z28.d, z0.d\n"
       "and z7.d, z29.d, z0.d\n"
       "and z8.d, z30.d, z0.d\n"
-      "asr z6.s, z6.s, #0x1f\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "sqadd z28.s, z28.s, z6.s\n"
-      "sqadd z29.s, z29.s, z7.s\n"
-      "sqadd z30.s, z30.s, z8.s\n"
       "and z9.d, z24.d, z0.d\n"
       "and z10.d, z25.d, z0.d\n"
       "and z4.d, z26.d, z0.d\n"
+      "and z5.d, z27.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z28.s, z28.s, z6.s\n"
+      "sqadd z29.s, z29.s, z7.s\n"
+      "sqadd z30.s, z30.s, z8.s\n"
       "sqadd z24.s, z24.s, z9.s\n"
       "sqadd z25.s, z25.s, z10.s\n"
       "sqadd z26.s, z26.s, z4.s\n"
-      "and z5.d, z27.d, z0.d\n"
-      "asr z5.s, z5.s, #0x1f\n"
       "sqadd z27.s, z27.s, z5.s\n"
       "55:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
+      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z31.h, z31.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z31.b, z31.b, z20.b\n"
-      "st1b { z31.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z28.s, z28.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z31.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z29.s, z29.s, z4.s\n"
       "smax z28.s, p2/M, z28.s, z5.s\n"
-      "add z30.s, z30.s, z4.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      "uzp1 z23.h, z23.h, z28.h\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
-      "add z24.s, z24.s, z4.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      "uzp1 z23.h, z23.h, z28.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
-      "add z25.s, z25.s, z4.s\n"
-      "uzp1 z28.h, z29.h, z30.h\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "add z26.s, z26.s, z4.s\n"
+      "uzp1 z28.h, z29.h, z30.h\n"
       "uzp1 z23.b, z23.b, z28.b\n"
-      "st1b { z23.b }, p1, [x20]\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
       "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "st1b { z23.b }, p1, [x21]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x19]\n"
+      "st1b { z24.b }, p1, [x20]\n"
+      "addvl x27, x27, #1\n"
       "56:"  // Height 4: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 44b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 58f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "57:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "58:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp
index 6b08d2834b..1e71806838 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -95,137 +95,137 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[output_ptr]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 8f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "cmp x26, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
+      "add x26, x26, #0x10\n"
       "bgt 7b\n"
       "8:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "addvl x9, x9, #4\n"
       "ble 9f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
-      "addvl x28, x28, #4\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "addvl x9, x9, #4\n"
       "ble 9f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
-      "addvl x28, x28, #4\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "addvl x9, x9, #4\n"
       "ble 9f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
+      "addvl x9, x9, #4\n"
       "9:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 4b\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
       "add z9.s, z9.s, z1.s\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "add z10.s, z10.s, z2.s\n"
       "add z11.s, z11.s, z3.s\n"
+      "addvl x14, x14, #4\n"
       "tbz %x[flags], #4, 10f\n"
       "ld1w { z0.s }, p2/Z, [x12]\n"
       "ld1w { z4.s }, p2/Z, [x13]\n"
@@ -234,20 +234,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 11f\n"
       "10:"  // Height 1: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "11:"  // Height 1: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -262,26 +262,26 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z11.s, z11.s, z7.s\n"
       "12:"  // Height 1: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
       "add z11.s, z11.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
@@ -289,29 +289,29 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "st1b { z8.b }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
       "13:"  // Height 1: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "15:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -319,165 +319,165 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "16:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "17:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 18f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 19f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 19f\n"
       "18:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "19:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 21f\n"
       "20:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
       "sdot z15.s, z7.b, z1.b[3]\n"
       "bgt 20b\n"
       "21:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
+      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "ble 22f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
+      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "ble 22f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
+      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "ble 22f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
+      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
       "sdot z15.s, z7.b, z1.b[3]\n"
       "22:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 17b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "add x24, x11, x20\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "add x23, x9, x19\n"
-      "add z12.s, z12.s, z0.s\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "add z9.s, z9.s, z1.s\n"
-      "addvl x11, x11, #4\n"
-      "add z13.s, z13.s, z1.s\n"
       "add z10.s, z10.s, z2.s\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "add z11.s, z11.s, z3.s\n"
+      "add z12.s, z12.s, z0.s\n"
+      "addvl x14, x14, #4\n"
+      "add z13.s, z13.s, z1.s\n"
       "add z14.s, z14.s, z2.s\n"
       "add z15.s, z15.s, z3.s\n"
       "tbz %x[flags], #4, 23f\n"
@@ -488,20 +488,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 24f\n"
       "23:"  // Height 2: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "24:"  // Height 2: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -516,92 +516,92 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
       "and z6.d, z10.d, z2.d\n"
+      "and z7.d, z11.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "and z7.d, z11.d, z3.d\n"
+      "sqadd z11.s, z11.s, z7.s\n"
       "and z4.d, z12.d, z0.d\n"
       "and z5.d, z13.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z12.s, z12.s, z4.s\n"
-      "sqadd z13.s, z13.s, z5.s\n"
       "and z6.d, z14.d, z2.d\n"
       "and z7.d, z15.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z12.s, z12.s, z4.s\n"
+      "sqadd z13.s, z13.s, z5.s\n"
       "sqadd z14.s, z14.s, z6.s\n"
       "sqadd z15.s, z15.s, z7.s\n"
       "25:"  // Height 2: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
       "add z11.s, z11.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
+      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
+      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
+      "add z13.s, z13.s, z4.s\n"
+      "add z14.s, z14.s, z4.s\n"
+      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z15.s, z15.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
       "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z12.s, p2/M, z12.s, z6.s\n"
+      "smin z13.s, p2/M, z13.s, z6.s\n"
+      "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
-      "smin z12.s, p2/M, z12.s, z6.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
-      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
-      "uzp1 z9.h, z10.h, z11.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
+      "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "add z13.s, z13.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
-      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
-      "smin z13.s, p2/M, z13.s, z6.s\n"
-      "add z14.s, z14.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smin z14.s, p2/M, z14.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "uzp1 z12.h, z12.h, z13.h\n"
       "smax z14.s, p2/M, z14.s, z5.s\n"
+      "uzp1 z12.h, z12.h, z13.h\n"
+      "st1b { z8.b }, p1, [x11]\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "uzp1 z13.h, z14.h, z15.h\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x23]\n"
+      "st1b { z12.b }, p1, [x24]\n"
+      "addvl x11, x11, #1\n"
       "26:"  // Height 2: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "28:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -613,99 +613,99 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "29:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "30:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 32f\n"
       "31:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "32:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 34f\n"
       "33:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "add x24, x24, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -714,22 +714,22 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[3]\n"
       "bgt 33b\n"
       "34:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -737,18 +737,18 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "ble 35f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -756,18 +756,18 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "ble 35f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -775,17 +775,17 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "ble 35f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -793,27 +793,27 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[3]\n"
       "sdot z19.s, z7.b, z2.b[3]\n"
       "35:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 30b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "add x23, x9, x19\n"
-      "add z12.s, z12.s, z0.s\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
-      "add z16.s, z16.s, z0.s\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
       "add z9.s, z9.s, z1.s\n"
-      "add z13.s, z13.s, z1.s\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "add z10.s, z10.s, z2.s\n"
       "add z11.s, z11.s, z3.s\n"
+      "addvl x14, x14, #4\n"
+      "add z12.s, z12.s, z0.s\n"
+      "add z13.s, z13.s, z1.s\n"
       "add z14.s, z14.s, z2.s\n"
       "add z15.s, z15.s, z3.s\n"
+      "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
@@ -825,20 +825,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 37f\n"
       "36:"  // Height 3: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "37:"  // Height 3: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -857,124 +857,124 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
       "and z6.d, z10.d, z2.d\n"
+      "and z7.d, z11.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "and z7.d, z11.d, z3.d\n"
+      "sqadd z11.s, z11.s, z7.s\n"
       "and z4.d, z12.d, z0.d\n"
       "and z5.d, z13.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z12.s, z12.s, z4.s\n"
-      "sqadd z13.s, z13.s, z5.s\n"
       "and z6.d, z14.d, z2.d\n"
       "and z7.d, z15.d, z3.d\n"
-      "and z4.d, z16.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z12.s, z12.s, z4.s\n"
+      "sqadd z13.s, z13.s, z5.s\n"
       "sqadd z14.s, z14.s, z6.s\n"
       "sqadd z15.s, z15.s, z7.s\n"
-      "sqadd z16.s, z16.s, z4.s\n"
+      "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
       "and z7.d, z19.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "38:"  // Height 3: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
       "add z11.s, z11.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
+      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
+      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
+      "add z13.s, z13.s, z4.s\n"
+      "add z14.s, z14.s, z4.s\n"
+      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
       "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z12.s, p2/M, z12.s, z6.s\n"
+      "smin z13.s, p2/M, z13.s, z6.s\n"
+      "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
-      "smin z12.s, p2/M, z12.s, z6.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
-      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
-      "uzp1 z9.h, z10.h, z11.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
+      "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "add z13.s, z13.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
-      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z13.s, p2/M, z13.s, z6.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
-      "add z14.s, z14.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z16.s, z16.s, z4.s\n"
-      "add z17.s, z17.s, z4.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smin z14.s, p2/M, z14.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      "uzp1 z12.h, z12.h, z13.h\n"
       "smax z14.s, p2/M, z14.s, z5.s\n"
+      "uzp1 z12.h, z12.h, z13.h\n"
+      "st1b { z8.b }, p1, [x11]\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
       "uzp1 z13.h, z14.h, z15.h\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x23]\n"
-      "add z18.s, z18.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z12.b }, p1, [x24]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x22]\n"
+      "st1b { z16.b }, p1, [x23]\n"
+      "addvl x11, x11, #1\n"
       "39:"  // Height 3: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "41:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -990,118 +990,118 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "42:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "43:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 44f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 45f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 45f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 45f\n"
       "44:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "45:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 47f\n"
       "46:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "sdot z20.s, z6.b, z3.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "sdot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1112,25 +1112,25 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[3]\n"
       "bgt 46b\n"
       "47:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -1140,20 +1140,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "ble 48f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -1163,20 +1163,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "ble 48f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -1186,19 +1186,19 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "ble 48f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1208,28 +1208,28 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[3]\n"
       "sdot z23.s, z7.b, z3.b[3]\n"
       "48:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 43b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "add x22, x23, x20\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "add x23, x9, x19\n"
-      "add z12.s, z12.s, z0.s\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
-      "add z16.s, z16.s, z0.s\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "add z9.s, z9.s, z1.s\n"
-      "addvl x11, x11, #4\n"
-      "add z13.s, z13.s, z1.s\n"
       "add z10.s, z10.s, z2.s\n"
+      "addvl x14, x14, #4\n"
       "add z11.s, z11.s, z3.s\n"
+      "add z12.s, z12.s, z0.s\n"
+      "add z13.s, z13.s, z1.s\n"
       "add z14.s, z14.s, z2.s\n"
       "add z15.s, z15.s, z3.s\n"
+      "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
@@ -1245,20 +1245,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 50f\n"
       "49:"  // Height 4: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "50:"  // Height 4: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -1281,156 +1281,156 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
       "and z6.d, z10.d, z2.d\n"
+      "and z7.d, z11.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "and z7.d, z11.d, z3.d\n"
+      "sqadd z11.s, z11.s, z7.s\n"
       "and z4.d, z12.d, z0.d\n"
       "and z5.d, z13.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z12.s, z12.s, z4.s\n"
-      "sqadd z13.s, z13.s, z5.s\n"
       "and z6.d, z14.d, z2.d\n"
       "and z7.d, z15.d, z3.d\n"
-      "and z4.d, z16.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z12.s, z12.s, z4.s\n"
+      "sqadd z13.s, z13.s, z5.s\n"
       "sqadd z14.s, z14.s, z6.s\n"
       "sqadd z15.s, z15.s, z7.s\n"
-      "sqadd z16.s, z16.s, z4.s\n"
+      "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
       "and z7.d, z19.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "and z4.d, z20.d, z0.d\n"
       "and z5.d, z21.d, z1.d\n"
       "and z6.d, z22.d, z2.d\n"
+      "and z7.d, z23.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z20.s, z20.s, z4.s\n"
       "sqadd z21.s, z21.s, z5.s\n"
       "sqadd z22.s, z22.s, z6.s\n"
-      "and z7.d, z23.d, z3.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z23.s, z23.s, z7.s\n"
       "51:"  // Height 4: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
       "add z11.s, z11.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
+      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
+      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
+      "add z13.s, z13.s, z4.s\n"
+      "add z14.s, z14.s, z4.s\n"
+      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
       "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z12.s, p2/M, z12.s, z6.s\n"
+      "smin z13.s, p2/M, z13.s, z6.s\n"
+      "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
-      "smin z12.s, p2/M, z12.s, z6.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
-      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
-      "uzp1 z9.h, z10.h, z11.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
+      "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "add z13.s, z13.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
-      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z13.s, p2/M, z13.s, z6.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
-      "add z14.s, z14.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z16.s, z16.s, z4.s\n"
-      "add z17.s, z17.s, z4.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smin z14.s, p2/M, z14.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      "uzp1 z12.h, z12.h, z13.h\n"
       "smax z14.s, p2/M, z14.s, z5.s\n"
+      "uzp1 z12.h, z12.h, z13.h\n"
+      "st1b { z8.b }, p1, [x11]\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
       "uzp1 z13.h, z14.h, z15.h\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x23]\n"
-      "add z18.s, z18.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z12.b }, p1, [x24]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
-      "add z21.s, z21.s, z4.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x22]\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "uzp1 z20.h, z20.h, z21.h\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "st1b { z16.b }, p1, [x23]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x21]\n"
+      "st1b { z20.b }, p1, [x22]\n"
+      "addvl x11, x11, #1\n"
       "52:"  // Height 4: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "54:"  // Height 5: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -1450,137 +1450,137 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "55:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "56:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 58f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 58f\n"
       "57:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "58:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 60f\n"
       "59:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
-      "sdot z17.s, z7.b, z2.b[0]\n"
-      "add x21, x21, #0x10\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
+      "add x25, x25, #0x10\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
+      "sdot z17.s, z7.b, z2.b[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
       "sdot z26.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
       "sdot z26.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
       "sdot z26.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1593,28 +1593,28 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[3]\n"
       "bgt 59b\n"
       "60:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -1626,22 +1626,22 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
       "ble 61f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -1653,22 +1653,22 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
       "ble 61f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -1680,21 +1680,21 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
       "ble 61f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1706,29 +1706,29 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[3]\n"
       "sdot z27.s, z7.b, z4.b[3]\n"
       "61:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 56b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "add x23, x24, x20\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "add x23, x9, x19\n"
-      "add z12.s, z12.s, z0.s\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
-      "add z16.s, z16.s, z0.s\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
       "add z9.s, z9.s, z1.s\n"
-      "add x20, x21, x19\n"
-      "add z13.s, z13.s, z1.s\n"
-      "addvl x11, x11, #4\n"
+      "addvl x14, x14, #4\n"
       "add z10.s, z10.s, z2.s\n"
       "add z11.s, z11.s, z3.s\n"
+      "add z12.s, z12.s, z0.s\n"
+      "add z13.s, z13.s, z1.s\n"
       "add z14.s, z14.s, z2.s\n"
       "add z15.s, z15.s, z3.s\n"
+      "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
@@ -1748,20 +1748,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 63f\n"
       "62:"  // Height 5: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "63:"  // Height 5: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -1788,191 +1788,191 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
       "and z6.d, z10.d, z2.d\n"
+      "and z7.d, z11.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "and z7.d, z11.d, z3.d\n"
+      "sqadd z11.s, z11.s, z7.s\n"
       "and z4.d, z12.d, z0.d\n"
       "and z5.d, z13.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z12.s, z12.s, z4.s\n"
-      "sqadd z13.s, z13.s, z5.s\n"
       "and z6.d, z14.d, z2.d\n"
       "and z7.d, z15.d, z3.d\n"
-      "and z4.d, z16.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z12.s, z12.s, z4.s\n"
+      "sqadd z13.s, z13.s, z5.s\n"
       "sqadd z14.s, z14.s, z6.s\n"
       "sqadd z15.s, z15.s, z7.s\n"
-      "sqadd z16.s, z16.s, z4.s\n"
+      "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
       "and z7.d, z19.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "and z4.d, z20.d, z0.d\n"
       "and z5.d, z21.d, z1.d\n"
       "and z6.d, z22.d, z2.d\n"
+      "and z7.d, z23.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z20.s, z20.s, z4.s\n"
       "sqadd z21.s, z21.s, z5.s\n"
       "sqadd z22.s, z22.s, z6.s\n"
-      "and z7.d, z23.d, z3.d\n"
+      "sqadd z23.s, z23.s, z7.s\n"
       "and z4.d, z24.d, z0.d\n"
       "and z5.d, z25.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z23.s, z23.s, z7.s\n"
-      "sqadd z24.s, z24.s, z4.s\n"
-      "sqadd z25.s, z25.s, z5.s\n"
       "and z6.d, z26.d, z2.d\n"
       "and z7.d, z27.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z24.s, z24.s, z4.s\n"
+      "sqadd z25.s, z25.s, z5.s\n"
       "sqadd z26.s, z26.s, z6.s\n"
       "sqadd z27.s, z27.s, z7.s\n"
       "64:"  // Height 5: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
       "add z11.s, z11.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
+      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
+      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
+      "add z13.s, z13.s, z4.s\n"
+      "add z14.s, z14.s, z4.s\n"
+      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
+      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
       "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z12.s, p2/M, z12.s, z6.s\n"
+      "smin z13.s, p2/M, z13.s, z6.s\n"
+      "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
-      "smin z12.s, p2/M, z12.s, z6.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
-      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
-      "uzp1 z9.h, z10.h, z11.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
+      "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "add z13.s, z13.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
-      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z13.s, p2/M, z13.s, z6.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
-      "add z14.s, z14.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z16.s, z16.s, z4.s\n"
-      "add z17.s, z17.s, z4.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smin z14.s, p2/M, z14.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      "uzp1 z12.h, z12.h, z13.h\n"
       "smax z14.s, p2/M, z14.s, z5.s\n"
+      "uzp1 z12.h, z12.h, z13.h\n"
+      "st1b { z8.b }, p1, [x11]\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
       "uzp1 z13.h, z14.h, z15.h\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x23]\n"
-      "add z18.s, z18.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z12.b }, p1, [x24]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
-      "add z21.s, z21.s, z4.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x22]\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x23]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z26.s, z26.s, z4.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x21]\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smax z25.s, p2/M, z25.s, z5.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x22]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x20]\n"
+      "st1b { z24.b }, p1, [x21]\n"
+      "addvl x11, x11, #1\n"
       "65:"  // Height 5: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x6\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x20, #0x6\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "67:"  // Height 6: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -1996,156 +1996,156 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "68:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "69:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 70f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 71f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 71f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 71f\n"
       "70:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "71:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 73f\n"
       "72:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
-      "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
       "add x25, x25, #0x10\n"
-      "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "add x24, x24, #0x10\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
+      "sdot z8.s, z6.b, z0.b[0]\n"
+      "sdot z12.s, z6.b, z1.b[0]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
+      "sdot z20.s, z6.b, z3.b[0]\n"
       "add x23, x23, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      "sdot z20.s, z6.b, z3.b[0]\n"
-      "add x21, x21, #0x10\n"
-      "sdot z17.s, z7.b, z2.b[0]\n"
-      "add x20, x20, #0x10\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
       "sdot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
+      "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
       "sdot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
       "sdot z26.s, z6.b, z4.b[0]\n"
       "sdot z30.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
       "sdot z31.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
       "sdot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
       "sdot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
       "sdot z26.s, z6.b, z4.b[1]\n"
       "sdot z30.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
       "sdot z31.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
       "sdot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
       "sdot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
       "sdot z26.s, z6.b, z4.b[2]\n"
       "sdot z30.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
       "sdot z31.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
       "sdot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
       "sdot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -2160,31 +2160,31 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z31.s, z7.b, z5.b[3]\n"
       "bgt 72b\n"
       "73:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
       "sdot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
       "sdot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -2198,24 +2198,24 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[0]\n"
       "sdot z31.s, z7.b, z5.b[0]\n"
       "ble 74f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
       "sdot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
       "sdot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -2229,24 +2229,24 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[1]\n"
       "sdot z31.s, z7.b, z5.b[1]\n"
       "ble 74f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
       "sdot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
       "sdot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -2260,23 +2260,23 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[2]\n"
       "sdot z31.s, z7.b, z5.b[2]\n"
       "ble 74f\n"
-      "ld1b { z6.b }, p2/Z, [x28]\n"
+      "ld1b { z6.b }, p2/Z, [x9]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
       "sdot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
       "sdot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -2290,30 +2290,30 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[3]\n"
       "sdot z31.s, z7.b, z5.b[3]\n"
       "74:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 69b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
+      "add x23, x24, x20\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
+      "add x20, x21, x20\n"
       "add z8.s, z8.s, z0.s\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
-      "add x23, x9, x19\n"
-      "add z12.s, z12.s, z0.s\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
-      "add z16.s, z16.s, z0.s\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
       "add z9.s, z9.s, z1.s\n"
-      "add x20, x21, x19\n"
-      "add z13.s, z13.s, z1.s\n"
-      "add x19, x20, x19\n"
       "add z10.s, z10.s, z2.s\n"
-      "addvl x11, x11, #4\n"
       "add z11.s, z11.s, z3.s\n"
+      "addvl x14, x14, #4\n"
+      "add z12.s, z12.s, z0.s\n"
+      "add z13.s, z13.s, z1.s\n"
       "add z14.s, z14.s, z2.s\n"
       "add z15.s, z15.s, z3.s\n"
+      "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
@@ -2337,20 +2337,20 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 76f\n"
       "75:"  // Height 6: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "76:"  // Height 6: parameters loaded
       ".inst 0x04a47508  // sqrdmulh z8.s, z8.s, z4.s\n"
@@ -2381,223 +2381,223 @@ void sve_hybrid_s8qs_dot_6x4VL (
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
       "and z6.d, z10.d, z2.d\n"
+      "and z7.d, z11.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z8.s, z8.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "and z7.d, z11.d, z3.d\n"
+      "sqadd z11.s, z11.s, z7.s\n"
       "and z4.d, z12.d, z0.d\n"
       "and z5.d, z13.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z12.s, z12.s, z4.s\n"
-      "sqadd z13.s, z13.s, z5.s\n"
       "and z6.d, z14.d, z2.d\n"
       "and z7.d, z15.d, z3.d\n"
-      "and z4.d, z16.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z12.s, z12.s, z4.s\n"
+      "sqadd z13.s, z13.s, z5.s\n"
       "sqadd z14.s, z14.s, z6.s\n"
       "sqadd z15.s, z15.s, z7.s\n"
-      "sqadd z16.s, z16.s, z4.s\n"
+      "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
       "and z7.d, z19.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "and z4.d, z20.d, z0.d\n"
       "and z5.d, z21.d, z1.d\n"
       "and z6.d, z22.d, z2.d\n"
+      "and z7.d, z23.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z20.s, z20.s, z4.s\n"
       "sqadd z21.s, z21.s, z5.s\n"
       "sqadd z22.s, z22.s, z6.s\n"
-      "and z7.d, z23.d, z3.d\n"
+      "sqadd z23.s, z23.s, z7.s\n"
       "and z4.d, z24.d, z0.d\n"
       "and z5.d, z25.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z23.s, z23.s, z7.s\n"
-      "sqadd z24.s, z24.s, z4.s\n"
-      "sqadd z25.s, z25.s, z5.s\n"
       "and z6.d, z26.d, z2.d\n"
       "and z7.d, z27.d, z3.d\n"
-      "and z4.d, z28.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z24.s, z24.s, z4.s\n"
+      "sqadd z25.s, z25.s, z5.s\n"
       "sqadd z26.s, z26.s, z6.s\n"
       "sqadd z27.s, z27.s, z7.s\n"
-      "sqadd z28.s, z28.s, z4.s\n"
+      "and z4.d, z28.d, z0.d\n"
       "and z5.d, z29.d, z1.d\n"
       "and z6.d, z30.d, z2.d\n"
       "and z7.d, z31.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z28.s, z28.s, z4.s\n"
       "sqadd z29.s, z29.s, z5.s\n"
       "sqadd z30.s, z30.s, z6.s\n"
       "sqadd z31.s, z31.s, z7.s\n"
       "77:"  // Height 6: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z8.s, z8.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
-      "add z8.s, z8.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880c  // srshl z12.s, p2/M, z12.s, z0.s\n"
       "add z11.s, z11.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
+      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
+      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
+      "add z13.s, z13.s, z4.s\n"
+      "add z14.s, z14.s, z4.s\n"
+      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
+      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
+      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      ".inst 0x4482883d  // srshl z29.s, p2/M, z29.s, z1.s\n"
+      ".inst 0x4482885e  // srshl z30.s, p2/M, z30.s, z2.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      ".inst 0x4482887f  // srshl z31.s, p2/M, z31.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z31.s, z31.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z8.s, p2/M, z8.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
       "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z12.s, p2/M, z12.s, z6.s\n"
+      "smin z13.s, p2/M, z13.s, z6.s\n"
+      "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z31.s, p2/M, z31.s, z6.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
-      "smin z12.s, p2/M, z12.s, z6.s\n"
       "uzp1 z8.h, z8.h, z9.h\n"
-      ".inst 0x4482882d  // srshl z13.s, p2/M, z13.s, z1.s\n"
-      "uzp1 z9.h, z10.h, z11.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
+      "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x9]\n"
-      "add z13.s, z13.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884e  // srshl z14.s, p2/M, z14.s, z2.s\n"
-      ".inst 0x4482886f  // srshl z15.s, p2/M, z15.s, z3.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z13.s, p2/M, z13.s, z6.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
-      "add z14.s, z14.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z16.s, z16.s, z4.s\n"
-      "add z17.s, z17.s, z4.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smin z14.s, p2/M, z14.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      "uzp1 z12.h, z12.h, z13.h\n"
       "smax z14.s, p2/M, z14.s, z5.s\n"
+      "uzp1 z12.h, z12.h, z13.h\n"
+      "st1b { z8.b }, p1, [x11]\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
       "uzp1 z13.h, z14.h, z15.h\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "uzp1 z12.b, z12.b, z13.b\n"
-      "st1b { z12.b }, p1, [x23]\n"
-      "add z18.s, z18.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      ".inst 0x44828835  // srshl z21.s, p2/M, z21.s, z1.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z12.b }, p1, [x24]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828856  // srshl z22.s, p2/M, z22.s, z2.s\n"
-      "add z21.s, z21.s, z4.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x22]\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      ".inst 0x44828877  // srshl z23.s, p2/M, z23.s, z3.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x23]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z26.s, z26.s, z4.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x21]\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
-      ".inst 0x4482883d  // srshl z29.s, p2/M, z29.s, z1.s\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smax z25.s, p2/M, z25.s, z5.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      "add z28.s, z28.s, z4.s\n"
-      "add z29.s, z29.s, z4.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x22]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      ".inst 0x4482885e  // srshl z30.s, p2/M, z30.s, z2.s\n"
-      "uzp1 z25.h, z26.h, z27.h\n"
       "smax z28.s, p2/M, z28.s, z5.s\n"
+      "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x20]\n"
-      "add z30.s, z30.s, z4.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
-      ".inst 0x4482887f  // srshl z31.s, p2/M, z31.s, z3.s\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
-      "uzp1 z28.h, z28.h, z29.h\n"
-      "add z31.s, z31.s, z4.s\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "smin z31.s, p2/M, z31.s, z6.s\n"
+      "uzp1 z28.h, z28.h, z29.h\n"
+      "st1b { z24.b }, p1, [x21]\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "uzp1 z29.h, z30.h, z31.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p1, [x19]\n"
+      "st1b { z28.b }, p1, [x20]\n"
+      "addvl x11, x11, #1\n"
       "78:"  // Height 6: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp
index 6aba002706..6041794bdb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -95,17 +95,17 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 27f\n"
       "beq 14f\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x9, %x[output_ptr]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -113,124 +113,124 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 8f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "cmp x26, #0x10\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
       "bgt 7b\n"
       "8:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "subs x26, x26, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
+      "addvl x9, x9, #8\n"
       "ble 9f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "addvl x9, x9, #8\n"
       "9:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 4b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
       "uzp1 z9.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
       "uzp1 z10.d, z10.d, z14.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
       "uzp1 z11.d, z11.d, z15.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "mov z15.d, z8.d\n"
-      "add z9.s, z9.s, z1.s\n"
       "add z15.s, z15.s, z0.s\n"
+      "addvl x14, x14, #4\n"
+      "add z9.s, z9.s, z1.s\n"
       "add z10.s, z10.s, z2.s\n"
       "add z11.s, z11.s, z3.s\n"
       "tbz %x[flags], #4, 10f\n"
@@ -241,20 +241,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 11f\n"
       "10:"  // Height 1: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "11:"  // Height 1: parameters loaded
       ".inst 0x04a475ef  // sqrdmulh z15.s, z15.s, z4.s\n"
@@ -269,26 +269,26 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z15.s, z15.s, z4.s\n"
       "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z11.s, z11.s, z7.s\n"
       "12:"  // Height 1: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z15.s, z15.s, z4.s\n"
       ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      "add z15.s, z15.s, z4.s\n"
       "add z9.s, z9.s, z4.s\n"
       "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
       "add z11.s, z11.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z15.s, p2/M, z15.s, z6.s\n"
       "smin z9.s, p2/M, z9.s, z6.s\n"
       "smin z10.s, p2/M, z10.s, z6.s\n"
@@ -296,29 +296,29 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
-      "smax z11.s, p2/M, z11.s, z5.s\n"
       "uzp1 z15.h, z15.h, z9.h\n"
+      "smax z11.s, p2/M, z11.s, z5.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z15.b, z15.b, z9.b\n"
-      "st1b { z15.b }, p1, [x9]\n"
-      "addvl x9, x9, #1\n"
+      "st1b { z15.b }, p1, [x11]\n"
+      "addvl x11, x11, #1\n"
       "13:"  // Height 1: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 2b\n"
       "b 80f\n"
       "14:"  // Height 2
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "15:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -326,133 +326,133 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "16:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "17:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 18f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 19f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 19f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 19f\n"
       "18:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "19:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 21f\n"
       "20:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "bgt 20b\n"
       "21:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
+      "addvl x9, x9, #8\n"
       "ble 22f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "addvl x9, x9, #8\n"
       "22:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 17b\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
-      "add x23, x9, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
       "uzp2 z10.d, z10.d, z14.d\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
+      "add x24, x11, x20\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
+      "addvl x14, x14, #4\n"
       "mov z15.d, z7.d\n"
       "add z15.s, z15.s, z0.s\n"
       "add z12.s, z12.s, z1.s\n"
@@ -470,20 +470,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 24f\n"
       "23:"  // Height 2: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "24:"  // Height 2: parameters loaded
       ".inst 0x04a475ef  // sqrdmulh z15.s, z15.s, z4.s\n"
@@ -498,92 +498,92 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "and z4.d, z15.d, z0.d\n"
       "and z5.d, z12.d, z1.d\n"
       "and z6.d, z13.d, z2.d\n"
+      "and z7.d, z14.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z15.s, z15.s, z4.s\n"
       "sqadd z12.s, z12.s, z5.s\n"
       "sqadd z13.s, z13.s, z6.s\n"
-      "and z7.d, z14.d, z3.d\n"
+      "sqadd z14.s, z14.s, z7.s\n"
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z14.s, z14.s, z7.s\n"
-      "sqadd z8.s, z8.s, z4.s\n"
-      "sqadd z9.s, z9.s, z5.s\n"
       "and z6.d, z10.d, z2.d\n"
       "and z7.d, z11.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z8.s, z8.s, z4.s\n"
+      "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
       "sqadd z11.s, z11.s, z7.s\n"
       "25:"  // Height 2: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z15.s, z15.s, z4.s\n"
       ".inst 0x4482882c  // srshl z12.s, p2/M, z12.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884d  // srshl z13.s, p2/M, z13.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add z15.s, z15.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
       "add z13.s, z13.s, z4.s\n"
+      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
+      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
       "add z14.s, z14.s, z4.s\n"
       "add z8.s, z8.s, z4.s\n"
+      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
+      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
+      "add z9.s, z9.s, z4.s\n"
+      "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z11.s, z11.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z15.s, p2/M, z15.s, z6.s\n"
       "smin z12.s, p2/M, z12.s, z6.s\n"
       "smin z13.s, p2/M, z13.s, z6.s\n"
       "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z8.s, p2/M, z8.s, z6.s\n"
+      "smin z9.s, p2/M, z9.s, z6.s\n"
+      "smin z10.s, p2/M, z10.s, z6.s\n"
+      "smin z11.s, p2/M, z11.s, z6.s\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smax z14.s, p2/M, z14.s, z5.s\n"
-      "smin z8.s, p2/M, z8.s, z6.s\n"
       "uzp1 z15.h, z15.h, z12.h\n"
-      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "uzp1 z12.h, z13.h, z14.h\n"
+      "smax z14.s, p2/M, z14.s, z5.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
+      "uzp1 z12.h, z13.h, z14.h\n"
       "uzp1 z15.b, z15.b, z12.b\n"
-      "st1b { z15.b }, p1, [x9]\n"
-      "add z9.s, z9.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      "smin z9.s, p2/M, z9.s, z6.s\n"
-      "add z10.s, z10.s, z4.s\n"
-      "add z11.s, z11.s, z4.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
-      "smin z10.s, p2/M, z10.s, z6.s\n"
-      "smin z11.s, p2/M, z11.s, z6.s\n"
-      "uzp1 z8.h, z8.h, z9.h\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
+      "uzp1 z8.h, z8.h, z9.h\n"
+      "st1b { z15.b }, p1, [x11]\n"
       "smax z11.s, p2/M, z11.s, z5.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x23]\n"
+      "st1b { z8.b }, p1, [x24]\n"
+      "addvl x11, x11, #1\n"
       "26:"  // Height 2: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 15b\n"
       "b 80f\n"
       "27:"  // Height 3
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "28:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -599,176 +599,176 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "29:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "30:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 31f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 32f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 32f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 32f\n"
       "31:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "32:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 34f\n"
       "33:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "bgt 33b\n"
       "34:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       "ble 35f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "35:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 30b\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
-      "add x23, x9, x19\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "addvl x11, x11, #4\n"
       "uzp2 z10.d, z10.d, z14.d\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
+      "add x24, x11, x20\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
+      "add x23, x24, x20\n"
+      "addvl x14, x14, #4\n"
       "uzp1 z16.d, z16.d, z20.d\n"
       "uzp1 z17.d, z17.d, z21.d\n"
       "uzp1 z18.d, z18.d, z22.d\n"
@@ -794,20 +794,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 37f\n"
       "36:"  // Height 3: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "37:"  // Height 3: parameters loaded
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
@@ -826,124 +826,124 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z12.d, z1.d\n"
       "and z6.d, z13.d, z2.d\n"
+      "and z7.d, z14.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z12.s, z12.s, z5.s\n"
       "sqadd z13.s, z13.s, z6.s\n"
-      "and z7.d, z14.d, z3.d\n"
+      "sqadd z14.s, z14.s, z7.s\n"
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z14.s, z14.s, z7.s\n"
-      "sqadd z8.s, z8.s, z4.s\n"
-      "sqadd z9.s, z9.s, z5.s\n"
       "and z6.d, z10.d, z2.d\n"
       "and z7.d, z11.d, z3.d\n"
-      "and z4.d, z16.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z8.s, z8.s, z4.s\n"
+      "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
       "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z16.s, z16.s, z4.s\n"
+      "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
       "and z7.d, z19.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "38:"  // Height 3: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x4482882c  // srshl z12.s, p2/M, z12.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884d  // srshl z13.s, p2/M, z13.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
       "add z13.s, z13.s, z4.s\n"
+      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
+      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
       "add z14.s, z14.s, z4.s\n"
       "add z8.s, z8.s, z4.s\n"
+      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
+      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
+      "add z9.s, z9.s, z4.s\n"
+      "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z11.s, z11.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z12.s, p2/M, z12.s, z6.s\n"
       "smin z13.s, p2/M, z13.s, z6.s\n"
       "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z8.s, p2/M, z8.s, z6.s\n"
+      "smin z9.s, p2/M, z9.s, z6.s\n"
+      "smin z10.s, p2/M, z10.s, z6.s\n"
+      "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smax z14.s, p2/M, z14.s, z5.s\n"
-      "smin z8.s, p2/M, z8.s, z6.s\n"
       "uzp1 z23.h, z23.h, z12.h\n"
-      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "uzp1 z12.h, z13.h, z14.h\n"
+      "smax z14.s, p2/M, z14.s, z5.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
+      "uzp1 z12.h, z13.h, z14.h\n"
       "uzp1 z23.b, z23.b, z12.b\n"
-      "st1b { z23.b }, p1, [x9]\n"
-      "add z9.s, z9.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z9.s, p2/M, z9.s, z6.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
-      "add z10.s, z10.s, z4.s\n"
-      "add z11.s, z11.s, z4.s\n"
-      "add z16.s, z16.s, z4.s\n"
-      "add z17.s, z17.s, z4.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
-      "smin z10.s, p2/M, z10.s, z6.s\n"
-      "smin z11.s, p2/M, z11.s, z6.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      "uzp1 z8.h, z8.h, z9.h\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
+      "uzp1 z8.h, z8.h, z9.h\n"
+      "st1b { z23.b }, p1, [x11]\n"
       "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x23]\n"
-      "add z18.s, z18.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z8.b }, p1, [x24]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x22]\n"
+      "st1b { z16.b }, p1, [x23]\n"
+      "addvl x11, x11, #1\n"
       "39:"  // Height 3: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 28b\n"
       "b 80f\n"
       "40:"  // Height 4
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "41:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -959,185 +959,185 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "42:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "43:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 44f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 45f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 45f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 45f\n"
       "44:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "45:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 47f\n"
       "46:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "add x22, x22, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "bgt 46b\n"
       "47:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       "ble 48f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "48:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 43b\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
-      "add x23, x9, x19\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
+      "add x24, x11, x20\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "uzp1 z15.d, z16.d, z20.d\n"
       "uzp2 z16.d, z16.d, z20.d\n"
+      "addvl x14, x14, #4\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
       "uzp1 z21.d, z18.d, z22.d\n"
@@ -1169,20 +1169,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 50f\n"
       "49:"  // Height 4: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "50:"  // Height 4: parameters loaded
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
@@ -1205,156 +1205,156 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z12.d, z1.d\n"
       "and z6.d, z13.d, z2.d\n"
+      "and z7.d, z14.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z12.s, z12.s, z5.s\n"
       "sqadd z13.s, z13.s, z6.s\n"
-      "and z7.d, z14.d, z3.d\n"
+      "sqadd z14.s, z14.s, z7.s\n"
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z14.s, z14.s, z7.s\n"
-      "sqadd z8.s, z8.s, z4.s\n"
-      "sqadd z9.s, z9.s, z5.s\n"
       "and z6.d, z10.d, z2.d\n"
       "and z7.d, z11.d, z3.d\n"
-      "and z4.d, z15.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z8.s, z8.s, z4.s\n"
+      "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
       "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z15.s, z15.s, z4.s\n"
+      "and z4.d, z15.d, z0.d\n"
       "and z5.d, z20.d, z1.d\n"
       "and z6.d, z21.d, z2.d\n"
       "and z7.d, z22.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z15.s, z15.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
       "sqadd z21.s, z21.s, z6.s\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
+      "and z7.d, z19.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "and z7.d, z19.d, z3.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "51:"  // Height 4: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x4482882c  // srshl z12.s, p2/M, z12.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884d  // srshl z13.s, p2/M, z13.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
       "add z13.s, z13.s, z4.s\n"
+      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
+      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
       "add z14.s, z14.s, z4.s\n"
       "add z8.s, z8.s, z4.s\n"
+      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
+      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
+      "add z9.s, z9.s, z4.s\n"
+      "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
+      "add z11.s, z11.s, z4.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
+      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z12.s, p2/M, z12.s, z6.s\n"
       "smin z13.s, p2/M, z13.s, z6.s\n"
       "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z8.s, p2/M, z8.s, z6.s\n"
+      "smin z9.s, p2/M, z9.s, z6.s\n"
+      "smin z10.s, p2/M, z10.s, z6.s\n"
+      "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smax z14.s, p2/M, z14.s, z5.s\n"
-      "smin z8.s, p2/M, z8.s, z6.s\n"
       "uzp1 z23.h, z23.h, z12.h\n"
-      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "uzp1 z12.h, z13.h, z14.h\n"
+      "smax z14.s, p2/M, z14.s, z5.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
+      "uzp1 z12.h, z13.h, z14.h\n"
       "uzp1 z23.b, z23.b, z12.b\n"
-      "st1b { z23.b }, p1, [x9]\n"
-      "add z9.s, z9.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
-      "smin z9.s, p2/M, z9.s, z6.s\n"
-      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
-      "add z10.s, z10.s, z4.s\n"
-      "add z11.s, z11.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
-      "smin z10.s, p2/M, z10.s, z6.s\n"
-      "smin z11.s, p2/M, z11.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "uzp1 z8.h, z8.h, z9.h\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
+      "uzp1 z8.h, z8.h, z9.h\n"
+      "st1b { z23.b }, p1, [x11]\n"
       "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
-      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x23]\n"
-      "add z21.s, z21.s, z4.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "uzp1 z15.h, z15.h, z20.h\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      "uzp1 z15.h, z15.h, z20.h\n"
+      "st1b { z8.b }, p1, [x24]\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
-      "add z17.s, z17.s, z4.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "add z18.s, z18.s, z4.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z15.b, z15.b, z20.b\n"
-      "st1b { z15.b }, p1, [x22]\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "st1b { z15.b }, p1, [x23]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
+      "st1b { z16.b }, p1, [x22]\n"
+      "addvl x11, x11, #1\n"
       "52:"  // Height 4: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 41b\n"
       "b 80f\n"
       "53:"  // Height 5
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "54:"  // Height 5: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -1378,115 +1378,115 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "55:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "56:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 58f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 58f\n"
       "57:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "58:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 60f\n"
       "59:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "cmp x26, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "add x24, x24, #0x10\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
       "trn1 z4.d, z5.d, z6.d\n"
-      "add x22, x22, #0x10\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1495,80 +1495,80 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "bgt 59b\n"
       "60:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "subs x26, x26, #0x8\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
       "ble 61f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1576,30 +1576,30 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "61:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 56b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
-      "add x23, x9, x19\n"
+      "add x24, x11, x20\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "add x20, x21, x19\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "addvl x11, x11, #4\n"
       "uzp2 z11.d, z11.d, z15.d\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "uzp1 z15.d, z16.d, z20.d\n"
       "uzp2 z16.d, z16.d, z20.d\n"
+      "add x21, x22, x20\n"
+      "addvl x14, x14, #4\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
       "uzp1 z21.d, z18.d, z22.d\n"
@@ -1639,20 +1639,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 63f\n"
       "62:"  // Height 5: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "63:"  // Height 5: parameters loaded
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -1679,191 +1679,191 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z12.d, z1.d\n"
       "and z6.d, z13.d, z2.d\n"
+      "and z7.d, z14.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z12.s, z12.s, z5.s\n"
       "sqadd z13.s, z13.s, z6.s\n"
-      "and z7.d, z14.d, z3.d\n"
+      "sqadd z14.s, z14.s, z7.s\n"
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z14.s, z14.s, z7.s\n"
-      "sqadd z8.s, z8.s, z4.s\n"
-      "sqadd z9.s, z9.s, z5.s\n"
       "and z6.d, z10.d, z2.d\n"
       "and z7.d, z11.d, z3.d\n"
-      "and z4.d, z15.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z8.s, z8.s, z4.s\n"
+      "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
       "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z15.s, z15.s, z4.s\n"
+      "and z4.d, z15.d, z0.d\n"
       "and z5.d, z20.d, z1.d\n"
       "and z6.d, z21.d, z2.d\n"
       "and z7.d, z22.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z15.s, z15.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
       "sqadd z21.s, z21.s, z6.s\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
+      "and z7.d, z19.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "and z7.d, z19.d, z3.d\n"
+      "sqadd z19.s, z19.s, z7.s\n"
       "and z4.d, z24.d, z0.d\n"
       "and z5.d, z25.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z19.s, z19.s, z7.s\n"
-      "sqadd z24.s, z24.s, z4.s\n"
-      "sqadd z25.s, z25.s, z5.s\n"
       "and z6.d, z26.d, z2.d\n"
       "and z7.d, z27.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z24.s, z24.s, z4.s\n"
+      "sqadd z25.s, z25.s, z5.s\n"
       "sqadd z26.s, z26.s, z6.s\n"
       "sqadd z27.s, z27.s, z7.s\n"
       "64:"  // Height 5: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x4482882c  // srshl z12.s, p2/M, z12.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884d  // srshl z13.s, p2/M, z13.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
       "add z13.s, z13.s, z4.s\n"
+      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
+      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
       "add z14.s, z14.s, z4.s\n"
       "add z8.s, z8.s, z4.s\n"
+      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
+      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
+      "add z9.s, z9.s, z4.s\n"
+      "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
+      "add z11.s, z11.s, z4.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
+      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
+      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z12.s, p2/M, z12.s, z6.s\n"
       "smin z13.s, p2/M, z13.s, z6.s\n"
       "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z8.s, p2/M, z8.s, z6.s\n"
+      "smin z9.s, p2/M, z9.s, z6.s\n"
+      "smin z10.s, p2/M, z10.s, z6.s\n"
+      "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smax z14.s, p2/M, z14.s, z5.s\n"
-      "smin z8.s, p2/M, z8.s, z6.s\n"
       "uzp1 z31.h, z31.h, z12.h\n"
-      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "uzp1 z12.h, z13.h, z14.h\n"
+      "smax z14.s, p2/M, z14.s, z5.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
+      "uzp1 z12.h, z13.h, z14.h\n"
       "uzp1 z31.b, z31.b, z12.b\n"
-      "st1b { z31.b }, p1, [x9]\n"
-      "add z9.s, z9.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
-      "smin z9.s, p2/M, z9.s, z6.s\n"
-      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
-      "add z10.s, z10.s, z4.s\n"
-      "add z11.s, z11.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
-      "smin z10.s, p2/M, z10.s, z6.s\n"
-      "smin z11.s, p2/M, z11.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "uzp1 z8.h, z8.h, z9.h\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
+      "uzp1 z8.h, z8.h, z9.h\n"
+      "st1b { z31.b }, p1, [x11]\n"
       "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
-      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x23]\n"
-      "add z21.s, z21.s, z4.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "uzp1 z15.h, z15.h, z20.h\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      "uzp1 z15.h, z15.h, z20.h\n"
+      "st1b { z8.b }, p1, [x24]\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
-      "add z17.s, z17.s, z4.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "add z18.s, z18.s, z4.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z15.b, z15.b, z20.b\n"
-      "st1b { z15.b }, p1, [x22]\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z15.b }, p1, [x23]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z26.s, z26.s, z4.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smax z25.s, p2/M, z25.s, z5.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x20]\n"
+      "st1b { z24.b }, p1, [x21]\n"
+      "addvl x11, x11, #1\n"
       "65:"  // Height 5: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 54b\n"
       "b 80f\n"
       "66:"  // Height 6
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x6\n"
+      "mov x14, %x[col_bias]\n"
       "ldr x13, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
-      "mov x11, %x[col_bias]\n"
       "ldr x12, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
-      "mov x9, %x[output_ptr]\n"
       "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x20, #0x6\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "mov x11, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "67:"  // Height 6: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x10\n"
       "mov z8.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z9.s, #0x0\n"
-      "whilelt p1.b, x19, x10\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "mov z12.s, #0x0\n"
@@ -1887,120 +1887,120 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "68:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "69:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 70f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 71f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 71f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 71f\n"
       "70:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "71:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 73f\n"
       "72:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x24, x24, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
-      "add x23, x23, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "add x20, x20, #0x10\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
+      ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -2009,81 +2009,81 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "bgt 72b\n"
       "73:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
       "trn1 z2.d, z3.d, z4.d\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
       "ble 74f\n"
-      "ld1b { z7.b }, p2/Z, [x28]\n"
+      "ld1b { z7.b }, p2/Z, [x9]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
+      "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n"
+      "addvl x9, x9, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -2091,33 +2091,33 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "74:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 69b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x11, x20\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "ld1w { z0.s }, p2/Z, [x11]\n"
-      "add x23, x9, x19\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "ld1w { z1.s }, p2/Z, [x11, #1, MUL VL]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "ld1w { z2.s }, p2/Z, [x11, #2, MUL VL]\n"
-      "add x22, x23, x19\n"
+      "add x23, x24, x20\n"
+      "ld1w { z0.s }, p2/Z, [x14]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "ld1w { z3.s }, p2/Z, [x11, #3, MUL VL]\n"
-      "add x21, x22, x19\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "add x20, x21, x19\n"
+      "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n"
+      "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x19, x20, x19\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "addvl x11, x11, #4\n"
+      "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n"
+      "add x22, x23, x20\n"
       "uzp1 z15.d, z16.d, z20.d\n"
       "uzp2 z16.d, z16.d, z20.d\n"
+      "add x21, x22, x20\n"
+      "add x20, x21, x20\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
+      "addvl x14, x14, #4\n"
       "uzp1 z21.d, z18.d, z22.d\n"
       "uzp2 z18.d, z18.d, z22.d\n"
       "uzp1 z22.d, z19.d, z23.d\n"
@@ -2163,20 +2163,20 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "ld1w { z2.s }, p2/Z, [x12, #2, MUL VL]\n"
       "ld1w { z6.s }, p2/Z, [x13, #2, MUL VL]\n"
       "ld1w { z3.s }, p2/Z, [x12, #3, MUL VL]\n"
-      "addvl x12, x12, #4\n"
       "ld1w { z7.s }, p2/Z, [x13, #3, MUL VL]\n"
+      "addvl x12, x12, #4\n"
       "addvl x13, x13, #4\n"
       "b 76f\n"
       "75:"  // Height 6: per layer parameters
-      "add x24, %x[qp], %[per_layer_right_shift]\n"
-      "ld1rw { z0.s }, p2/Z, [x24]\n"
+      "add x26, %x[qp], %[per_layer_right_shift]\n"
+      "add x25, %x[qp], %[per_layer_mul]\n"
+      "ld1rw { z0.s }, p2/Z, [x26]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       "mov z1.d, z0.d\n"
-      "add x24, %x[qp], %[per_layer_mul]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
-      "mov z2.d, z0.d\n"
-      "mov z3.d, z0.d\n"
       "mov z5.d, z4.d\n"
+      "mov z2.d, z0.d\n"
       "mov z6.d, z4.d\n"
+      "mov z3.d, z0.d\n"
       "mov z7.d, z4.d\n"
       "76:"  // Height 6: parameters loaded
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -2207,223 +2207,223 @@ void sve_hybrid_s8qs_mmla_6x4VL (
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z12.d, z1.d\n"
       "and z6.d, z13.d, z2.d\n"
+      "and z7.d, z14.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z12.s, z12.s, z5.s\n"
       "sqadd z13.s, z13.s, z6.s\n"
-      "and z7.d, z14.d, z3.d\n"
+      "sqadd z14.s, z14.s, z7.s\n"
       "and z4.d, z8.d, z0.d\n"
       "and z5.d, z9.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z14.s, z14.s, z7.s\n"
-      "sqadd z8.s, z8.s, z4.s\n"
-      "sqadd z9.s, z9.s, z5.s\n"
       "and z6.d, z10.d, z2.d\n"
       "and z7.d, z11.d, z3.d\n"
-      "and z4.d, z15.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z8.s, z8.s, z4.s\n"
+      "sqadd z9.s, z9.s, z5.s\n"
       "sqadd z10.s, z10.s, z6.s\n"
       "sqadd z11.s, z11.s, z7.s\n"
-      "sqadd z15.s, z15.s, z4.s\n"
+      "and z4.d, z15.d, z0.d\n"
       "and z5.d, z20.d, z1.d\n"
       "and z6.d, z21.d, z2.d\n"
       "and z7.d, z22.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z15.s, z15.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
       "sqadd z21.s, z21.s, z6.s\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z1.d\n"
       "and z6.d, z18.d, z2.d\n"
+      "and z7.d, z19.d, z3.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "and z7.d, z19.d, z3.d\n"
+      "sqadd z19.s, z19.s, z7.s\n"
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z28.d, z1.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z19.s, z19.s, z7.s\n"
-      "sqadd z23.s, z23.s, z4.s\n"
-      "sqadd z28.s, z28.s, z5.s\n"
       "and z6.d, z29.d, z2.d\n"
       "and z7.d, z30.d, z3.d\n"
-      "and z4.d, z24.d, z0.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z23.s, z23.s, z4.s\n"
+      "sqadd z28.s, z28.s, z5.s\n"
       "sqadd z29.s, z29.s, z6.s\n"
       "sqadd z30.s, z30.s, z7.s\n"
-      "sqadd z24.s, z24.s, z4.s\n"
+      "and z4.d, z24.d, z0.d\n"
       "and z5.d, z25.d, z1.d\n"
       "and z6.d, z26.d, z2.d\n"
       "and z7.d, z27.d, z3.d\n"
+      "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
+      "sqadd z24.s, z24.s, z4.s\n"
       "sqadd z25.s, z25.s, z5.s\n"
       "sqadd z26.s, z26.s, z6.s\n"
       "sqadd z27.s, z27.s, z7.s\n"
       "77:"  // Height 6: no shift correction
+      "add x25, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x25]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x24, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x24]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x4482882c  // srshl z12.s, p2/M, z12.s, z1.s\n"
-      "add x24, %x[qp], %[minval]\n"
       ".inst 0x4482884d  // srshl z13.s, p2/M, z13.s, z2.s\n"
-      "ld1rw { z5.s }, p2/Z, [x24]\n"
-      "add x24, %x[qp], %[maxval]\n"
-      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
-      "ld1rw { z6.s }, p2/Z, [x24]\n"
-      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z12.s, z12.s, z4.s\n"
       "add z13.s, z13.s, z4.s\n"
+      ".inst 0x4482886e  // srshl z14.s, p2/M, z14.s, z3.s\n"
+      ".inst 0x44828808  // srshl z8.s, p2/M, z8.s, z0.s\n"
       "add z14.s, z14.s, z4.s\n"
       "add z8.s, z8.s, z4.s\n"
+      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
+      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
+      "add z9.s, z9.s, z4.s\n"
+      "add z10.s, z10.s, z4.s\n"
+      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
+      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
+      "add z11.s, z11.s, z4.s\n"
+      "add z15.s, z15.s, z4.s\n"
+      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
+      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
+      "add z20.s, z20.s, z4.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      ".inst 0x4482883c  // srshl z28.s, p2/M, z28.s, z1.s\n"
+      ".inst 0x4482885d  // srshl z29.s, p2/M, z29.s, z2.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      ".inst 0x4482887e  // srshl z30.s, p2/M, z30.s, z3.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
+      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
+      "add x25, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x25]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x25, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x25]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z12.s, p2/M, z12.s, z6.s\n"
       "smin z13.s, p2/M, z13.s, z6.s\n"
       "smin z14.s, p2/M, z14.s, z6.s\n"
+      "smin z8.s, p2/M, z8.s, z6.s\n"
+      "smin z9.s, p2/M, z9.s, z6.s\n"
+      "smin z10.s, p2/M, z10.s, z6.s\n"
+      "smin z11.s, p2/M, z11.s, z6.s\n"
+      "smin z15.s, p2/M, z15.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z12.s, p2/M, z12.s, z5.s\n"
       "smax z13.s, p2/M, z13.s, z5.s\n"
-      "smax z14.s, p2/M, z14.s, z5.s\n"
-      "smin z8.s, p2/M, z8.s, z6.s\n"
       "uzp1 z31.h, z31.h, z12.h\n"
-      ".inst 0x44828829  // srshl z9.s, p2/M, z9.s, z1.s\n"
-      "uzp1 z12.h, z13.h, z14.h\n"
+      "smax z14.s, p2/M, z14.s, z5.s\n"
       "smax z8.s, p2/M, z8.s, z5.s\n"
+      "uzp1 z12.h, z13.h, z14.h\n"
       "uzp1 z31.b, z31.b, z12.b\n"
-      "st1b { z31.b }, p1, [x9]\n"
-      "add z9.s, z9.s, z4.s\n"
-      "addvl x9, x9, #1\n"
-      ".inst 0x4482884a  // srshl z10.s, p2/M, z10.s, z2.s\n"
-      ".inst 0x4482886b  // srshl z11.s, p2/M, z11.s, z3.s\n"
-      ".inst 0x4482880f  // srshl z15.s, p2/M, z15.s, z0.s\n"
-      "smin z9.s, p2/M, z9.s, z6.s\n"
-      ".inst 0x44828834  // srshl z20.s, p2/M, z20.s, z1.s\n"
-      "add z10.s, z10.s, z4.s\n"
-      "add z11.s, z11.s, z4.s\n"
-      "add z15.s, z15.s, z4.s\n"
-      "add z20.s, z20.s, z4.s\n"
       "smax z9.s, p2/M, z9.s, z5.s\n"
-      "smin z10.s, p2/M, z10.s, z6.s\n"
-      "smin z11.s, p2/M, z11.s, z6.s\n"
-      "smin z15.s, p2/M, z15.s, z6.s\n"
-      "uzp1 z8.h, z8.h, z9.h\n"
       "smax z10.s, p2/M, z10.s, z5.s\n"
+      "uzp1 z8.h, z8.h, z9.h\n"
+      "st1b { z31.b }, p1, [x11]\n"
       "smax z11.s, p2/M, z11.s, z5.s\n"
       "smax z15.s, p2/M, z15.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
-      ".inst 0x44828855  // srshl z21.s, p2/M, z21.s, z2.s\n"
       "uzp1 z9.h, z10.h, z11.h\n"
-      ".inst 0x44828876  // srshl z22.s, p2/M, z22.s, z3.s\n"
       "uzp1 z8.b, z8.b, z9.b\n"
-      "st1b { z8.b }, p1, [x23]\n"
-      "add z21.s, z21.s, z4.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "uzp1 z15.h, z15.h, z20.h\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      ".inst 0x44828831  // srshl z17.s, p2/M, z17.s, z1.s\n"
+      "uzp1 z15.h, z15.h, z20.h\n"
+      "st1b { z8.b }, p1, [x24]\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
-      ".inst 0x44828852  // srshl z18.s, p2/M, z18.s, z2.s\n"
-      "add z17.s, z17.s, z4.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
-      "add z18.s, z18.s, z4.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z15.b, z15.b, z20.b\n"
-      "st1b { z15.b }, p1, [x22]\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      ".inst 0x44828873  // srshl z19.s, p2/M, z19.s, z3.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      ".inst 0x4482883c  // srshl z28.s, p2/M, z28.s, z1.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z28.s, z28.s, z4.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      ".inst 0x4482885d  // srshl z29.s, p2/M, z29.s, z2.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z15.b }, p1, [x23]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
-      "smax z28.s, p2/M, z28.s, z5.s\n"
-      "add z29.s, z29.s, z4.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482887e  // srshl z30.s, p2/M, z30.s, z3.s\n"
-      "uzp1 z23.h, z23.h, z28.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z30.s, z30.s, z4.s\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      ".inst 0x44828839  // srshl z25.s, p2/M, z25.s, z1.s\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smax z28.s, p2/M, z28.s, z5.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
+      "uzp1 z23.h, z23.h, z28.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482885a  // srshl z26.s, p2/M, z26.s, z2.s\n"
-      "uzp1 z28.h, z29.h, z30.h\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
+      "uzp1 z28.h, z29.h, z30.h\n"
       "uzp1 z23.b, z23.b, z28.b\n"
-      "st1b { z23.b }, p1, [x20]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      ".inst 0x4482887b  // srshl z27.s, p2/M, z27.s, z3.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "add z27.s, z27.s, z4.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z23.b }, p1, [x21]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x19]\n"
+      "st1b { z24.b }, p1, [x20]\n"
+      "addvl x11, x11, #1\n"
       "78:"  // Height 6: Writeback done
       "decw x10, ALL, MUL #4\n"
       "cmp x10, XZR\n"
       "bgt 67b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 80f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 79f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "79:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "80:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp
index e0fea96ef3..51e9aa1b40 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,23 +87,23 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 21f\n"
       "beq 11f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
       "b 4f\n"
       "3:"  // Height 1: no accumulate
       "mov z8.s, #0x0\n"
@@ -111,87 +111,87 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x25, x25, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x26, x26, #0x4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z11.s, z7.b, z0.b\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z11.s, z7.b, z0.b\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "bne 5b\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "10:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 62f\n"
       "11:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "12:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 13f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
       "b 14f\n"
       "13:"  // Height 2: no accumulate
       "mov z8.s, #0x0\n"
@@ -203,112 +203,112 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "14:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 17f\n"
       "16:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "17:"  // Height 2: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 19f\n"
       "18:"  // Height 2: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x25, x25, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x26, x26, #0x4\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "subs x26, x26, #0x4\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x4\n"
+      "add x25, x25, #0x4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z11.s, z7.b, z0.b\n"
       "sdot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 18b\n"
       "19:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "sdot z11.s, z7.b, z0.b\n"
       "sdot z15.s, z7.b, z1.b\n"
       "bne 15b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
       "20:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 12b\n"
       "b 62f\n"
       "21:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "22:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
       "b 24f\n"
       "23:"  // Height 3: no accumulate
       "mov z8.s, #0x0\n"
@@ -324,74 +324,74 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "24:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "25:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 26f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 27f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 27f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 27f\n"
       "26:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "27:"  // Height 3: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 29f\n"
       "28:"  // Height 3: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x23, x23, #0x4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x24, x24, #0x4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
       "sdot z11.s, z7.b, z0.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "sdot z15.s, z7.b, z1.b\n"
       "sdot z19.s, z7.b, z2.b\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 28b\n"
       "29:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
@@ -399,61 +399,61 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "sdot z15.s, z7.b, z1.b\n"
       "sdot z19.s, z7.b, z2.b\n"
       "bne 25b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
       "30:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 22b\n"
       "b 62f\n"
       "31:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "32:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
       "b 34f\n"
       "33:"  // Height 4: no accumulate
       "mov z8.s, #0x0\n"
@@ -473,86 +473,86 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "34:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "35:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 37f\n"
       "36:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "37:"  // Height 4: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 39f\n"
       "38:"  // Height 4: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "sdot z17.s, z7.b, z2.b\n"
       "sdot z21.s, z7.b, z3.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
       "sdot z22.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "sdot z11.s, z7.b, z0.b\n"
       "sdot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "sdot z19.s, z7.b, z2.b\n"
       "sdot z23.s, z7.b, z3.b\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 38b\n"
       "39:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
       "sdot z21.s, z7.b, z3.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
@@ -562,71 +562,71 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "sdot z19.s, z7.b, z2.b\n"
       "sdot z23.s, z7.b, z3.b\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
       "40:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 32b\n"
       "b 62f\n"
       "41:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "42:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x20]\n"
-      "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x21]\n"
+      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
       "b 44f\n"
       "43:"  // Height 5: no accumulate
       "mov z8.s, #0x0\n"
@@ -650,98 +650,98 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "44:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 47f\n"
       "46:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "47:"  // Height 5: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 49f\n"
       "48:"  // Height 5: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "sdot z24.s, z6.b, z4.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
-      "add x21, x21, #0x4\n"
+      "add x22, x22, #0x4\n"
       "sdot z21.s, z7.b, z3.b\n"
       "sdot z25.s, z7.b, z4.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
       "sdot z22.s, z6.b, z3.b\n"
       "sdot z26.s, z6.b, z4.b\n"
       "sdot z11.s, z7.b, z0.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "sdot z15.s, z7.b, z1.b\n"
       "sdot z19.s, z7.b, z2.b\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
       "sdot z23.s, z7.b, z3.b\n"
       "sdot z27.s, z7.b, z4.b\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 48b\n"
       "49:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "sdot z24.s, z6.b, z4.b\n"
       "sdot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
       "sdot z21.s, z7.b, z3.b\n"
       "sdot z25.s, z7.b, z4.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
@@ -753,84 +753,84 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "sdot z23.s, z7.b, z3.b\n"
       "sdot z27.s, z7.b, z4.b\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x20]\n"
-      "st1w { z25.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x21]\n"
+      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
       "50:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 42b\n"
       "b 62f\n"
       "51:"  // Height 6
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "52:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x20]\n"
-      "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n"
-      "ld1w { z28.s }, p3/Z, [x19]\n"
-      "ld1w { z29.s }, p2/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p1/Z, [x19, #2, MUL VL]\n"
-      "ld1w { z31.s }, p0/Z, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x21]\n"
+      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p3/Z, [x20]\n"
+      "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n"
       "b 54f\n"
       "53:"  // Height 6: no accumulate
       "mov z8.s, #0x0\n"
@@ -858,110 +858,110 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "54:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "55:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 56f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 57f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 57f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 57f\n"
       "56:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "57:"  // Height 6: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 59f\n"
       "58:"  // Height 6: Multiply loop: Main loop
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "sdot z24.s, z6.b, z4.b\n"
       "sdot z28.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
+      "add x22, x22, #0x4\n"
       "add x21, x21, #0x4\n"
-      "add x20, x20, #0x4\n"
       "sdot z17.s, z7.b, z2.b\n"
       "sdot z21.s, z7.b, z3.b\n"
       "sdot z25.s, z7.b, z4.b\n"
       "sdot z29.s, z7.b, z5.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
       "sdot z22.s, z6.b, z3.b\n"
       "sdot z26.s, z6.b, z4.b\n"
       "sdot z30.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "sdot z11.s, z7.b, z0.b\n"
       "sdot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "sdot z19.s, z7.b, z2.b\n"
       "sdot z23.s, z7.b, z3.b\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
       "sdot z27.s, z7.b, z4.b\n"
       "sdot z31.s, z7.b, z5.b\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 58b\n"
       "59:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "sdot z8.s, z6.b, z0.b\n"
       "sdot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "sdot z16.s, z6.b, z2.b\n"
       "sdot z20.s, z6.b, z3.b\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "sdot z24.s, z6.b, z4.b\n"
       "sdot z28.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b\n"
       "sdot z13.s, z7.b, z1.b\n"
       "sdot z17.s, z7.b, z2.b\n"
       "sdot z21.s, z7.b, z3.b\n"
       "sdot z25.s, z7.b, z4.b\n"
       "sdot z29.s, z7.b, z5.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b\n"
       "sdot z14.s, z6.b, z1.b\n"
       "sdot z18.s, z6.b, z2.b\n"
@@ -975,57 +975,57 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx (
       "sdot z27.s, z7.b, z4.b\n"
       "sdot z31.s, z7.b, z5.b\n"
       "bne 55b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x20]\n"
-      "st1w { z25.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x20, #3, MUL VL]\n"
-      "st1w { z28.s }, p3, [x19]\n"
-      "st1w { z29.s }, p2, [x19, #1, MUL VL]\n"
-      "st1w { z30.s }, p1, [x19, #2, MUL VL]\n"
-      "st1w { z31.s }, p0, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x21]\n"
+      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
+      "st1w { z28.s }, p3, [x20]\n"
+      "st1w { z29.s }, p2, [x20, #1, MUL VL]\n"
+      "st1w { z30.s }, p1, [x20, #2, MUL VL]\n"
+      "st1w { z31.s }, p0, [x20, #3, MUL VL]\n"
       "60:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 52b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 62f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "61:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "62:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp
index dc5b7a33f4..b3d2e6b271 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,23 +87,23 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 23f\n"
       "beq 12f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
       "b 4f\n"
       "3:"  // Height 1: no accumulate
       "mov z8.s, #0x0\n"
@@ -111,148 +111,148 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "cmp x26, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
+      "add x26, x26, #0x10\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
-      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
-      "addvl x9, x9, #4\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
+      "addvl x10, x10, #4\n"
       "10:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 5b\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "11:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 68f\n"
       "12:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "13:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
       "b 15f\n"
       "14:"  // Height 2: no accumulate
       "mov z8.s, #0x0\n"
@@ -264,197 +264,197 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "15:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 18f\n"
       "17:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "18:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 20f\n"
       "19:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
       "sdot z15.s, z7.b, z1.b[3]\n"
       "bgt 19b\n"
       "20:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
+      "addvl x10, x10, #4\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
+      "addvl x10, x10, #4\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
+      "addvl x10, x10, #4\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
+      "addvl x10, x10, #4\n"
       "sdot z11.s, z7.b, z0.b[3]\n"
       "sdot z15.s, z7.b, z1.b[3]\n"
       "21:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
       "22:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 13b\n"
       "b 68f\n"
       "23:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "24:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
       "b 26f\n"
       "25:"  // Height 3: no accumulate
       "mov z8.s, #0x0\n"
@@ -470,99 +470,99 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "26:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "27:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 28f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 29f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 29f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 29f\n"
       "28:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "29:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 31f\n"
       "30:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "add x24, x24, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -571,22 +571,22 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[3]\n"
       "bgt 30b\n"
       "31:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -594,18 +594,18 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -613,18 +613,18 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -632,17 +632,17 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -650,65 +650,65 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z15.s, z7.b, z1.b[3]\n"
       "sdot z19.s, z7.b, z2.b[3]\n"
       "32:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 27b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "33:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 24b\n"
       "b 68f\n"
       "34:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "35:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "b 37f\n"
       "36:"  // Height 4: no accumulate
       "mov z8.s, #0x0\n"
@@ -728,118 +728,118 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "37:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "38:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 39f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 40f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 40f\n"
       "39:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "40:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 42f\n"
       "41:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "sdot z20.s, z6.b, z3.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
       "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
-      "sdot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -850,25 +850,25 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[3]\n"
       "bgt 41b\n"
       "42:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -878,20 +878,20 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -901,20 +901,20 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -924,19 +924,19 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -946,75 +946,75 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z19.s, z7.b, z2.b[3]\n"
       "sdot z23.s, z7.b, z3.b[3]\n"
       "43:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 38b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
       "44:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 35b\n"
       "b 68f\n"
       "45:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "46:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x20]\n"
-      "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x21]\n"
+      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
       "b 48f\n"
       "47:"  // Height 5: no accumulate
       "mov z8.s, #0x0\n"
@@ -1038,137 +1038,137 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "48:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 51f\n"
       "50:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "51:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 53f\n"
       "52:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
-      "sdot z17.s, z7.b, z2.b[0]\n"
-      "add x21, x21, #0x10\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
+      "add x25, x25, #0x10\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
+      "sdot z17.s, z7.b, z2.b[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
       "sdot z26.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
       "sdot z26.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
       "sdot z26.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1181,28 +1181,28 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[3]\n"
       "bgt 52b\n"
       "53:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -1214,22 +1214,22 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -1241,22 +1241,22 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -1268,21 +1268,21 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1294,88 +1294,88 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z23.s, z7.b, z3.b[3]\n"
       "sdot z27.s, z7.b, z4.b[3]\n"
       "54:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "55:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 46b\n"
       "b 68f\n"
       "56:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "57:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x20]\n"
-      "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x19]\n"
-      "ld1w { z29.s }, p3/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x19, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x21]\n"
+      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x20]\n"
+      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
       "b 59f\n"
       "58:"  // Height 6: no accumulate
       "mov z8.s, #0x0\n"
@@ -1403,156 +1403,156 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "59:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "60:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 62f\n"
       "61:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "62:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 64f\n"
       "63:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
-      "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
       "add x25, x25, #0x10\n"
-      "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "add x24, x24, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
+      "sdot z8.s, z6.b, z0.b[0]\n"
+      "sdot z12.s, z6.b, z1.b[0]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
+      "sdot z20.s, z6.b, z3.b[0]\n"
       "add x23, x23, #0x10\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      "sdot z20.s, z6.b, z3.b[0]\n"
-      "add x21, x21, #0x10\n"
-      "sdot z17.s, z7.b, z2.b[0]\n"
-      "add x20, x20, #0x10\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
       "sdot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
+      "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
       "sdot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
       "sdot z22.s, z6.b, z3.b[0]\n"
       "sdot z26.s, z6.b, z4.b[0]\n"
       "sdot z30.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[0]\n"
       "sdot z15.s, z7.b, z1.b[0]\n"
       "sdot z19.s, z7.b, z2.b[0]\n"
       "sdot z23.s, z7.b, z3.b[0]\n"
       "sdot z27.s, z7.b, z4.b[0]\n"
       "sdot z31.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
       "sdot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
       "sdot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
       "sdot z22.s, z6.b, z3.b[1]\n"
       "sdot z26.s, z6.b, z4.b[1]\n"
       "sdot z30.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z15.s, z7.b, z1.b[1]\n"
       "sdot z19.s, z7.b, z2.b[1]\n"
       "sdot z23.s, z7.b, z3.b[1]\n"
       "sdot z27.s, z7.b, z4.b[1]\n"
       "sdot z31.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
       "sdot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
       "sdot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
       "sdot z22.s, z6.b, z3.b[2]\n"
       "sdot z26.s, z6.b, z4.b[2]\n"
       "sdot z30.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[2]\n"
       "sdot z15.s, z7.b, z1.b[2]\n"
       "sdot z19.s, z7.b, z2.b[2]\n"
       "sdot z23.s, z7.b, z3.b[2]\n"
       "sdot z27.s, z7.b, z4.b[2]\n"
       "sdot z31.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
       "sdot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
       "sdot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1567,31 +1567,31 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z31.s, z7.b, z5.b[3]\n"
       "bgt 63b\n"
       "64:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "sdot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "sdot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "sdot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "sdot z16.s, z6.b, z2.b[0]\n"
       "sdot z20.s, z6.b, z3.b[0]\n"
       "sdot z24.s, z6.b, z4.b[0]\n"
       "sdot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sdot z9.s, z7.b, z0.b[0]\n"
+      "sdot z13.s, z7.b, z1.b[0]\n"
       "sdot z17.s, z7.b, z2.b[0]\n"
       "sdot z21.s, z7.b, z3.b[0]\n"
       "sdot z25.s, z7.b, z4.b[0]\n"
       "sdot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z14.s, z6.b, z1.b[0]\n"
       "sdot z18.s, z6.b, z2.b[0]\n"
@@ -1605,24 +1605,24 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[0]\n"
       "sdot z31.s, z7.b, z5.b[0]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[1]\n"
       "sdot z16.s, z6.b, z2.b[1]\n"
       "sdot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[1]\n"
       "sdot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[1]\n"
       "sdot z13.s, z7.b, z1.b[1]\n"
       "sdot z17.s, z7.b, z2.b[1]\n"
       "sdot z21.s, z7.b, z3.b[1]\n"
       "sdot z25.s, z7.b, z4.b[1]\n"
       "sdot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[1]\n"
       "sdot z14.s, z6.b, z1.b[1]\n"
       "sdot z18.s, z6.b, z2.b[1]\n"
@@ -1636,24 +1636,24 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[1]\n"
       "sdot z31.s, z7.b, z5.b[1]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "sdot z12.s, z6.b, z1.b[2]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
       "sdot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "sdot z24.s, z6.b, z4.b[2]\n"
       "sdot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[2]\n"
       "sdot z13.s, z7.b, z1.b[2]\n"
       "sdot z17.s, z7.b, z2.b[2]\n"
       "sdot z21.s, z7.b, z3.b[2]\n"
       "sdot z25.s, z7.b, z4.b[2]\n"
       "sdot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[2]\n"
       "sdot z14.s, z6.b, z1.b[2]\n"
       "sdot z18.s, z6.b, z2.b[2]\n"
@@ -1667,23 +1667,23 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[2]\n"
       "sdot z31.s, z7.b, z5.b[2]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "sdot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "sdot z12.s, z6.b, z1.b[3]\n"
       "sdot z16.s, z6.b, z2.b[3]\n"
       "sdot z20.s, z6.b, z3.b[3]\n"
       "sdot z24.s, z6.b, z4.b[3]\n"
       "sdot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "sdot z9.s, z7.b, z0.b[3]\n"
       "sdot z13.s, z7.b, z1.b[3]\n"
       "sdot z17.s, z7.b, z2.b[3]\n"
       "sdot z21.s, z7.b, z3.b[3]\n"
       "sdot z25.s, z7.b, z4.b[3]\n"
       "sdot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "sdot z10.s, z6.b, z0.b[3]\n"
       "sdot z14.s, z6.b, z1.b[3]\n"
       "sdot z18.s, z6.b, z2.b[3]\n"
@@ -1697,61 +1697,61 @@ void sve_hybrid_s8s32_dot_6x4VL (
       "sdot z27.s, z7.b, z4.b[3]\n"
       "sdot z31.s, z7.b, z5.b[3]\n"
       "65:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 60b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
-      "st1w { z28.s }, p4, [x19]\n"
-      "st1w { z29.s }, p3, [x19, #1, MUL VL]\n"
-      "st1w { z30.s }, p2, [x19, #2, MUL VL]\n"
-      "st1w { z31.s }, p1, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z28.s }, p4, [x20]\n"
+      "st1w { z29.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z30.s }, p2, [x20, #2, MUL VL]\n"
+      "st1w { z31.s }, p1, [x20, #3, MUL VL]\n"
       "66:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 57b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 68f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "67:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "68:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp
index c3abb203ca..350425647a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,25 +87,25 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 23f\n"
       "beq 12f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
@@ -123,154 +123,154 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "cmp x26, #0x10\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "subs x26, x26, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "ble 10f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "10:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 5b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
-      "st1w { z8.s }, p4, [x28]\n"
       "uzp1 z9.d, z9.d, z13.d\n"
+      "st1w { z8.s }, p4, [x9]\n"
       "uzp1 z10.d, z10.d, z14.d\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z11.d, z11.d, z15.d\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "11:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 68f\n"
       "12:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "13:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
@@ -286,179 +286,179 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "15:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 18f\n"
       "17:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "18:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 20f\n"
       "19:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "bgt 19b\n"
       "20:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "ble 21f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "21:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x24]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "22:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 13b\n"
       "b 68f\n"
       "23:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "24:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
@@ -486,239 +486,239 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "26:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "27:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 28f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 29f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 29f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 29f\n"
       "28:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "29:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 31f\n"
       "30:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "bgt 30b\n"
       "31:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       "ble 32f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "32:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 27b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z16.d, z16.d, z20.d\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp1 z17.d, z17.d, z21.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
       "uzp1 z18.d, z18.d, z22.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp1 z19.d, z19.d, z23.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "33:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 24b\n"
       "b 68f\n"
       "34:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "35:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
@@ -742,263 +742,263 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "37:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "38:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 39f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 40f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 40f\n"
       "39:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "40:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 42f\n"
       "41:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
-      "add x22, x22, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "bgt 41b\n"
       "42:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       "ble 43f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       "43:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 38b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x21]\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
       "44:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 35b\n"
       "b 68f\n"
       "45:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "46:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z25.s }, p4/Z, [x20]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x21]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
+      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
       "zip2 z23.d, z24.d, z23.d\n"
+      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
@@ -1034,115 +1034,115 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "48:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 51f\n"
       "50:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "51:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 53f\n"
       "52:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "cmp x26, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "add x24, x24, #0x10\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
-      "add x22, x22, #0x10\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1151,80 +1151,80 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "bgt 52b\n"
       "53:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "subs x26, x26, #0x8\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
       "ble 54f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1232,127 +1232,127 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "54:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "addvl x28, x28, #4\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
       "uzp1 z24.d, z24.d, z28.d\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp1 z25.d, z25.d, z29.d\n"
-      "st1w { z16.s }, p4, [x21]\n"
       "uzp1 z26.d, z26.d, z30.d\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "uzp1 z27.d, z27.d, z31.d\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "55:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 46b\n"
       "b 68f\n"
       "56:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "57:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z25.s }, p4/Z, [x20]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x21]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
-      "ld1w { z28.s }, p4/Z, [x19]\n"
+      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x20]\n"
       "zip2 z23.d, z24.d, z23.d\n"
-      "ld1w { z29.s }, p3/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x19, #2, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
-      "ld1w { z31.s }, p1/Z, [x19, #3, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
+      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip2 z29.d, z26.d, z29.d\n"
       "zip1 z26.d, z27.d, z30.d\n"
       "zip2 z30.d, z27.d, z30.d\n"
@@ -1385,120 +1385,120 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "59:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "60:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 62f\n"
       "61:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "62:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 64f\n"
       "63:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x24, x24, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
-      "add x23, x23, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
-      "add x20, x20, #0x10\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
+      ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1507,81 +1507,81 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "bgt 63b\n"
       "64:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
       "trn1 z2.d, z3.d, z4.d\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45079808  // smmla z8.s, z0.b, z7.b\n"
       ".inst 0x45079850  // smmla z16.s, z2.b, z7.b\n"
       ".inst 0x45079898  // smmla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x4506980c  // smmla z12.s, z0.b, z6.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
       ".inst 0x4506989c  // smmla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079809  // smmla z9.s, z0.b, z7.b\n"
       ".inst 0x45079851  // smmla z17.s, z2.b, z7.b\n"
       ".inst 0x45079899  // smmla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x4506980d  // smmla z13.s, z0.b, z6.b\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
       ".inst 0x4506989d  // smmla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507980a  // smmla z10.s, z0.b, z7.b\n"
       ".inst 0x45079852  // smmla z18.s, z2.b, z7.b\n"
       ".inst 0x4507989a  // smmla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x4506980e  // smmla z14.s, z0.b, z6.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x4506989e  // smmla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45079853  // smmla z19.s, z2.b, z7.b\n"
       ".inst 0x4507989b  // smmla z27.s, z4.b, z7.b\n"
       ".inst 0x4506980f  // smmla z15.s, z0.b, z6.b\n"
       ".inst 0x45069857  // smmla z23.s, z2.b, z6.b\n"
       ".inst 0x4506989f  // smmla z31.s, z4.b, z6.b\n"
       "ble 65f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45079828  // smmla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45079870  // smmla z16.s, z3.b, z7.b\n"
       ".inst 0x450798b8  // smmla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x4506982c  // smmla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x45069874  // smmla z20.s, z3.b, z6.b\n"
       ".inst 0x450698bc  // smmla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45079829  // smmla z9.s, z1.b, z7.b\n"
       ".inst 0x45079871  // smmla z17.s, z3.b, z7.b\n"
       ".inst 0x450798b9  // smmla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x4506982d  // smmla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45069875  // smmla z21.s, z3.b, z6.b\n"
       ".inst 0x450698bd  // smmla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x4507982a  // smmla z10.s, z1.b, z7.b\n"
       ".inst 0x45079872  // smmla z18.s, z3.b, z7.b\n"
       ".inst 0x450798ba  // smmla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45069876  // smmla z22.s, z3.b, z6.b\n"
       ".inst 0x450698be  // smmla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x4507982b  // smmla z11.s, z1.b, z7.b\n"
       ".inst 0x45079873  // smmla z19.s, z3.b, z7.b\n"
       ".inst 0x450798bb  // smmla z27.s, z5.b, z7.b\n"
@@ -1589,85 +1589,85 @@ void sve_hybrid_s8s32_mmla_6x4VL (
       ".inst 0x45069877  // smmla z23.s, z3.b, z6.b\n"
       ".inst 0x450698bf  // smmla z31.s, z5.b, z6.b\n"
       "65:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 60b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "add x19, x20, x19, LSL #2\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
       "uzp1 z23.d, z24.d, z28.d\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp2 z24.d, z24.d, z28.d\n"
-      "st1w { z16.s }, p4, [x21]\n"
       "uzp1 z28.d, z25.d, z29.d\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "uzp2 z25.d, z25.d, z29.d\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
       "uzp1 z29.d, z26.d, z30.d\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
       "uzp2 z26.d, z26.d, z30.d\n"
-      "st1w { z23.s }, p4, [x20]\n"
       "uzp1 z30.d, z27.d, z31.d\n"
-      "st1w { z28.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
       "uzp2 z27.d, z27.d, z31.d\n"
-      "st1w { z29.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z30.s }, p1, [x20, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x19]\n"
-      "st1w { z25.s }, p3, [x19, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x19, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x19, #3, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z23.s }, p4, [x21]\n"
+      "st1w { z28.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z29.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z30.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x20]\n"
+      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
       "66:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 57b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 68f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "67:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "68:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp
index be6d5b901d..79bd563a4b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -85,165 +85,165 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z15.b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x23, #0x10\n"
       "udot z17.s, z5.b, z0.b[0]\n"
+      "udot z18.s, z6.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z19.s, z7.b, z0.b[0]\n"
       "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "udot z18.s, z6.b, z0.b[0]\n"
+      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "udot z19.s, z7.b, z0.b[0]\n"
-      "udot z16.s, z8.b, z0.b[1]\n"
+      "udot z17.s, z9.b, z0.b[1]\n"
+      "udot z18.s, z10.b, z0.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      "udot z17.s, z9.b, z0.b[1]\n"
+      "udot z19.s, z4.b, z0.b[1]\n"
       "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
-      "udot z18.s, z10.b, z0.b[1]\n"
       "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
-      "udot z19.s, z4.b, z0.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "udot z16.s, z5.b, z0.b[2]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "udot z17.s, z6.b, z0.b[2]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "udot z19.s, z8.b, z0.b[2]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "udot z16.s, z9.b, z0.b[3]\n"
       "udot z17.s, z10.b, z0.b[3]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "udot z18.s, z4.b, z0.b[3]\n"
       "udot z19.s, z5.b, z0.b[3]\n"
+      "add x24, x24, #0x10\n"
       "tbnz %x[flags], #31, 8f\n"
       "udot z11.s, z0.b, z15.b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      "subs x25, x25, #0x4\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
-      "udot z16.s, z4.b, z0.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "udot z16.s, z4.b, z0.b[0]\n"
       "udot z17.s, z5.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "udot z18.s, z6.b, z0.b[0]\n"
       "udot z19.s, z7.b, z0.b[0]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z17.s, z9.b, z0.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z17.s, z9.b, z0.b[1]\n"
       "udot z18.s, z10.b, z0.b[1]\n"
-      "addvl x28, x28, #4\n"
       "udot z19.s, z4.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z17.s, z6.b, z0.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z17.s, z6.b, z0.b[2]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
-      "addvl x28, x28, #4\n"
       "udot z19.s, z8.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "ble 10f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "udot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "udot z16.s, z9.b, z0.b[3]\n"
       "udot z17.s, z10.b, z0.b[3]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "udot z18.s, z4.b, z0.b[3]\n"
       "udot z19.s, z5.b, z0.b[3]\n"
+      "addvl x28, x28, #4\n"
       "10:"  // Height 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "udot z11.s, z0.b, z15.b\n"
       "11:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "tbnz %x[flags], #31, 12f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z1.s }, p2/Z, [x19]\n"
-      "neg z1.s, p2/M, z1.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z1.s }, p2/Z, [x20]\n"
       "uaddv d11, p0, z11.s\n"
       "mov z11.s, z11.s[0]\n"
+      "neg z1.s, p2/M, z1.s\n"
       "mul z11.s, p2/M, z11.s, z1.s\n"
       "12:"  // Height 1: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z16.s, z16.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
       ".inst 0x04a47631  // sqrdmulh z17.s, z17.s, z4.s\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x04a47652  // sqrdmulh z18.s, z18.s, z4.s\n"
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 13f\n"
@@ -254,26 +254,26 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "13:"  // Height 1: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
       "add z19.s, z19.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
@@ -281,31 +281,31 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "addvl x26, x26, #1\n"
+      "st1b { z16.b }, p1, [x27]\n"
+      "addvl x27, x27, #1\n"
       "14:"  // Height 1: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 2b\n"
       "b 58f\n"
       "15:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z15.b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -313,68 +313,68 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "18:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "20:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 23f\n"
       "21:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
       "udot z20.s, z4.b, z1.b[0]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
       "udot z21.s, z5.b, z1.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "udot z18.s, z6.b, z0.b[0]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "udot z22.s, z6.b, z1.b[0]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
       "udot z19.s, z7.b, z0.b[0]\n"
+      "udot z23.s, z7.b, z1.b[0]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      "udot z23.s, z7.b, z1.b[0]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "udot z16.s, z8.b, z0.b[1]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
       "udot z20.s, z8.b, z1.b[1]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "udot z17.s, z9.b, z0.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "udot z21.s, z9.b, z1.b[1]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "udot z18.s, z10.b, z0.b[1]\n"
       "udot z22.s, z10.b, z1.b[1]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "udot z19.s, z4.b, z0.b[1]\n"
       "udot z23.s, z4.b, z1.b[1]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "udot z16.s, z5.b, z0.b[2]\n"
       "udot z20.s, z5.b, z1.b[2]\n"
       "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       "udot z17.s, z6.b, z0.b[2]\n"
       "udot z21.s, z6.b, z1.b[2]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
@@ -393,69 +393,69 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z11.s, z0.b, z15.b\n"
       "udot z12.s, z1.b, z15.b\n"
       "22:"  // Height 2: Multiply loop: unique 3: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 21b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
+      "udot z20.s, z4.b, z1.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "udot z20.s, z4.b, z1.b[0]\n"
-      "addvl x28, x28, #4\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
       "udot z21.s, z5.b, z1.b[0]\n"
       "udot z18.s, z6.b, z0.b[0]\n"
       "udot z22.s, z6.b, z1.b[0]\n"
+      "addvl x28, x28, #4\n"
       "udot z19.s, z7.b, z0.b[0]\n"
       "udot z23.s, z7.b, z1.b[0]\n"
       "ble 24f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z20.s, z8.b, z1.b[1]\n"
       "udot z17.s, z9.b, z0.b[1]\n"
-      "addvl x28, x28, #4\n"
       "udot z21.s, z9.b, z1.b[1]\n"
       "udot z18.s, z10.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "udot z22.s, z10.b, z1.b[1]\n"
       "udot z19.s, z4.b, z0.b[1]\n"
       "udot z23.s, z4.b, z1.b[1]\n"
       "ble 24f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
+      "udot z20.s, z5.b, z1.b[2]\n"
       "udot z17.s, z6.b, z0.b[2]\n"
-      "addvl x28, x28, #4\n"
       "udot z21.s, z6.b, z1.b[2]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "udot z22.s, z7.b, z1.b[2]\n"
       "udot z19.s, z8.b, z0.b[2]\n"
       "udot z23.s, z8.b, z1.b[2]\n"
       "ble 24f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "udot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "udot z16.s, z9.b, z0.b[3]\n"
       "udot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
       "udot z17.s, z10.b, z0.b[3]\n"
-      "addvl x28, x28, #4\n"
       "udot z21.s, z10.b, z1.b[3]\n"
       "udot z18.s, z4.b, z0.b[3]\n"
       "udot z22.s, z4.b, z1.b[3]\n"
+      "addvl x28, x28, #4\n"
       "udot z19.s, z5.b, z0.b[3]\n"
       "udot z23.s, z5.b, z1.b[3]\n"
       "24:"  // Height 2: Multiply loop: multiply skip
@@ -463,49 +463,49 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z11.s, z0.b, z15.b\n"
       "udot z12.s, z1.b, z15.b\n"
       "25:"  // Height 2: Multiply loop: unique 4: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 18b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
       "tbnz %x[flags], #31, 26f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z2.s }, p2/Z, [x19]\n"
-      "neg z2.s, p2/M, z2.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z2.s }, p2/Z, [x20]\n"
       "uaddv d11, p0, z11.s\n"
-      "uaddv d12, p0, z12.s\n"
       "mov z11.s, z11.s[0]\n"
+      "uaddv d12, p0, z12.s\n"
+      "neg z2.s, p2/M, z2.s\n"
       "mov z12.s, z12.s[0]\n"
       "mul z11.s, p2/M, z11.s, z2.s\n"
       "mul z12.s, p2/M, z12.s, z2.s\n"
       "26:"  // Height 2: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z16.s, z16.s, z0.s\n"
       "add z17.s, z17.s, z1.s\n"
+      "addvl x10, x10, #4\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       "add z20.s, z20.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z21.s, z21.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z22.s, z22.s, z2.s\n"
       "add z23.s, z23.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -518,97 +518,97 @@ void sve_hybrid_u8qa_dot_4x4VL (
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
       "tbz %x[flags], #5, 27f\n"
       "and z4.d, z16.d, z0.d\n"
-      "and z5.d, z17.d, z0.d\n"
-      "and z6.d, z18.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
-      "sqadd z17.s, z17.s, z5.s\n"
-      "sqadd z18.s, z18.s, z6.s\n"
+      "and z5.d, z17.d, z0.d\n"
+      "and z6.d, z18.d, z0.d\n"
       "and z7.d, z19.d, z0.d\n"
       "and z8.d, z20.d, z0.d\n"
       "and z9.d, z21.d, z0.d\n"
+      "and z10.d, z22.d, z0.d\n"
+      "and z4.d, z23.d, z0.d\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z17.s, z17.s, z5.s\n"
+      "sqadd z18.s, z18.s, z6.s\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "sqadd z20.s, z20.s, z8.s\n"
       "sqadd z21.s, z21.s, z9.s\n"
-      "and z10.d, z22.d, z0.d\n"
-      "and z4.d, z23.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
       "sqadd z22.s, z22.s, z10.s\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "27:"  // Height 2: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
+      "st1b { z20.b }, p1, [x23]\n"
+      "addvl x27, x27, #1\n"
       "28:"  // Height 2: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 16b\n"
       "b 58f\n"
       "29:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "mov z15.b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -620,83 +620,83 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "34:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 37f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "udot z20.s, z4.b, z1.b[0]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
       "udot z24.s, z4.b, z2.b[0]\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
       "udot z21.s, z5.b, z1.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "udot z25.s, z5.b, z2.b[0]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "udot z18.s, z6.b, z0.b[0]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "udot z22.s, z6.b, z1.b[0]\n"
-      "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
       "udot z26.s, z6.b, z2.b[0]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "udot z19.s, z7.b, z0.b[0]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
+      "addvl x28, x28, #16\n"
       "udot z23.s, z7.b, z1.b[0]\n"
       "udot z27.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
       "udot z16.s, z8.b, z0.b[1]\n"
       "udot z20.s, z8.b, z1.b[1]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       "udot z24.s, z8.b, z2.b[1]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
       "udot z17.s, z9.b, z0.b[1]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       "udot z21.s, z9.b, z1.b[1]\n"
       "udot z25.s, z9.b, z2.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "udot z18.s, z10.b, z0.b[1]\n"
       "udot z22.s, z10.b, z1.b[1]\n"
       "udot z26.s, z10.b, z2.b[1]\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "udot z19.s, z4.b, z0.b[1]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n"
       "udot z23.s, z4.b, z1.b[1]\n"
       "udot z27.s, z4.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "udot z16.s, z5.b, z0.b[2]\n"
       "udot z20.s, z5.b, z1.b[2]\n"
       "udot z24.s, z5.b, z2.b[2]\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "udot z17.s, z6.b, z0.b[2]\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n"
       "udot z21.s, z6.b, z1.b[2]\n"
       "udot z25.s, z6.b, z2.b[2]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
@@ -722,26 +722,26 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z12.s, z1.b, z15.b\n"
       "udot z13.s, z2.b, z15.b\n"
       "36:"  // Height 3: Multiply loop: unique 5: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 35b\n"
       "37:"  // Height 3: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "udot z20.s, z4.b, z1.b[0]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
       "udot z24.s, z4.b, z2.b[0]\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
       "udot z21.s, z5.b, z1.b[0]\n"
       "udot z25.s, z5.b, z2.b[0]\n"
+      "addvl x28, x28, #4\n"
       "udot z18.s, z6.b, z0.b[0]\n"
       "udot z22.s, z6.b, z1.b[0]\n"
       "udot z26.s, z6.b, z2.b[0]\n"
@@ -750,16 +750,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z27.s, z7.b, z2.b[0]\n"
       "ble 38f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z8.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z20.s, z8.b, z1.b[1]\n"
+      "udot z24.s, z8.b, z2.b[1]\n"
       "udot z17.s, z9.b, z0.b[1]\n"
       "udot z21.s, z9.b, z1.b[1]\n"
+      "addvl x28, x28, #4\n"
       "udot z25.s, z9.b, z2.b[1]\n"
       "udot z18.s, z10.b, z0.b[1]\n"
       "udot z22.s, z10.b, z1.b[1]\n"
@@ -769,16 +769,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z27.s, z4.b, z2.b[1]\n"
       "ble 38f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z5.b, z2.b[2]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z20.s, z5.b, z1.b[2]\n"
+      "udot z24.s, z5.b, z2.b[2]\n"
       "udot z17.s, z6.b, z0.b[2]\n"
       "udot z21.s, z6.b, z1.b[2]\n"
+      "addvl x28, x28, #4\n"
       "udot z25.s, z6.b, z2.b[2]\n"
       "udot z18.s, z7.b, z0.b[2]\n"
       "udot z22.s, z7.b, z1.b[2]\n"
@@ -788,16 +788,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z27.s, z8.b, z2.b[2]\n"
       "ble 38f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "udot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "udot z16.s, z9.b, z0.b[3]\n"
       "udot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z9.b, z2.b[3]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z24.s, z9.b, z2.b[3]\n"
       "udot z17.s, z10.b, z0.b[3]\n"
       "udot z21.s, z10.b, z1.b[3]\n"
       "udot z25.s, z10.b, z2.b[3]\n"
+      "addvl x28, x28, #4\n"
       "udot z18.s, z4.b, z0.b[3]\n"
       "udot z22.s, z4.b, z1.b[3]\n"
       "udot z26.s, z4.b, z2.b[3]\n"
@@ -810,48 +810,48 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z12.s, z1.b, z15.b\n"
       "udot z13.s, z2.b, z15.b\n"
       "39:"  // Height 3: Multiply loop: unique 6: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 32b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
-      "add x21, x22, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
+      "add x22, x23, x20\n"
       "tbnz %x[flags], #31, 40f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z3.s }, p2/Z, [x19]\n"
-      "neg z3.s, p2/M, z3.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z3.s }, p2/Z, [x20]\n"
       "uaddv d11, p0, z11.s\n"
+      "mov z11.s, z11.s[0]\n"
       "uaddv d12, p0, z12.s\n"
       "uaddv d13, p0, z13.s\n"
-      "mov z11.s, z11.s[0]\n"
       "mov z12.s, z12.s[0]\n"
       "mov z13.s, z13.s[0]\n"
+      "neg z3.s, p2/M, z3.s\n"
       "mul z11.s, p2/M, z11.s, z3.s\n"
       "mul z12.s, p2/M, z12.s, z3.s\n"
       "mul z13.s, p2/M, z13.s, z3.s\n"
       "40:"  // Height 3: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z16.s, z16.s, z0.s\n"
@@ -863,8 +863,8 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "add z22.s, z22.s, z2.s\n"
       "add z23.s, z23.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -883,131 +883,131 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z0.d\n"
       "and z6.d, z18.d, z0.d\n"
+      "and z7.d, z19.d, z0.d\n"
+      "and z8.d, z20.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "and z7.d, z19.d, z0.d\n"
-      "and z8.d, z20.d, z0.d\n"
-      "and z9.d, z21.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "sqadd z20.s, z20.s, z8.s\n"
-      "sqadd z21.s, z21.s, z9.s\n"
+      "and z9.d, z21.d, z0.d\n"
       "and z10.d, z22.d, z0.d\n"
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z22.s, z22.s, z10.s\n"
-      "sqadd z23.s, z23.s, z4.s\n"
-      "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
+      "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
+      "sqadd z21.s, z21.s, z9.s\n"
+      "sqadd z22.s, z22.s, z10.s\n"
+      "sqadd z23.s, z23.s, z4.s\n"
+      "sqadd z24.s, z24.s, z5.s\n"
       "sqadd z25.s, z25.s, z6.s\n"
       "sqadd z26.s, z26.s, z7.s\n"
       "sqadd z27.s, z27.s, z8.s\n"
       "41:"  // Height 3: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x23]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x21]\n"
+      "st1b { z24.b }, p1, [x22]\n"
+      "addvl x27, x27, #1\n"
       "42:"  // Height 3: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 30b\n"
       "b 58f\n"
       "43:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z14.s, #0x0\n"
-      "mov x19, #0x4\n"
       "mov z15.b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -1023,66 +1023,66 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "46:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "48:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 51f\n"
       "49:"  // Height 4: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "add x24, x24, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "ld1rqb { z3.b }, p0/Z, [x21]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "add x23, x23, #0x10\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "udot z20.s, z4.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x20]\n"
-      "add x21, x21, #0x10\n"
-      "udot z24.s, z4.b, z2.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x20, x20, #0x10\n"
-      "udot z21.s, z5.b, z1.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "udot z25.s, z5.b, z2.b[0]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "udot z24.s, z4.b, z2.b[0]\n"
       "udot z28.s, z4.b, z3.b[0]\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
+      "udot z21.s, z5.b, z1.b[0]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n"
+      "udot z25.s, z5.b, z2.b[0]\n"
       "udot z29.s, z5.b, z3.b[0]\n"
       "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "udot z18.s, z6.b, z0.b[0]\n"
       "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #16\n"
+      "udot z18.s, z6.b, z0.b[0]\n"
       "udot z22.s, z6.b, z1.b[0]\n"
+      "addvl x28, x28, #16\n"
       "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n"
       "udot z26.s, z6.b, z2.b[0]\n"
       "udot z30.s, z6.b, z3.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "add x21, x21, #0x10\n"
       "udot z19.s, z7.b, z0.b[0]\n"
       "udot z23.s, z7.b, z1.b[0]\n"
       "udot z27.s, z7.b, z2.b[0]\n"
@@ -1147,27 +1147,27 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z13.s, z2.b, z15.b\n"
       "udot z14.s, z3.b, z15.b\n"
       "50:"  // Height 4: Multiply loop: unique 7: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 49b\n"
       "51:"  // Height 4: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z0.b }, p0/Z, [x24]\n"
+      "ld1rqb { z1.b }, p0/Z, [x23]\n"
+      "subs x25, x25, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "ld1rqb { z3.b }, p0/Z, [x21]\n"
       "ld1b { z4.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
       "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x23]\n"
       "udot z16.s, z4.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x22]\n"
-      "udot z17.s, z5.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x21]\n"
-      "ld1rqb { z3.b }, p0/Z, [x20]\n"
       "udot z20.s, z4.b, z1.b[0]\n"
       "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z21.s, z5.b, z1.b[0]\n"
       "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
       "udot z24.s, z4.b, z2.b[0]\n"
       "udot z28.s, z4.b, z3.b[0]\n"
+      "udot z17.s, z5.b, z0.b[0]\n"
+      "udot z21.s, z5.b, z1.b[0]\n"
+      "addvl x28, x28, #4\n"
       "udot z25.s, z5.b, z2.b[0]\n"
       "udot z29.s, z5.b, z3.b[0]\n"
       "udot z18.s, z6.b, z0.b[0]\n"
@@ -1180,16 +1180,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z31.s, z7.b, z3.b[0]\n"
       "ble 52f\n"
       "ld1b { z8.b }, p2/Z, [x28]\n"
-      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z8.b, z1.b[1]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z8.b, z0.b[1]\n"
       "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z8.b, z2.b[1]\n"
       "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z20.s, z8.b, z1.b[1]\n"
+      "udot z24.s, z8.b, z2.b[1]\n"
       "udot z28.s, z8.b, z3.b[1]\n"
       "udot z17.s, z9.b, z0.b[1]\n"
+      "addvl x28, x28, #4\n"
       "udot z21.s, z9.b, z1.b[1]\n"
       "udot z25.s, z9.b, z2.b[1]\n"
       "udot z29.s, z9.b, z3.b[1]\n"
@@ -1203,16 +1203,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z31.s, z4.b, z3.b[1]\n"
       "ble 52f\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x4\n"
-      "udot z20.s, z5.b, z1.b[2]\n"
+      "subs x25, x25, #0x4\n"
+      "udot z16.s, z5.b, z0.b[2]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z5.b, z2.b[2]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z20.s, z5.b, z1.b[2]\n"
+      "udot z24.s, z5.b, z2.b[2]\n"
       "udot z28.s, z5.b, z3.b[2]\n"
       "udot z17.s, z6.b, z0.b[2]\n"
+      "addvl x28, x28, #4\n"
       "udot z21.s, z6.b, z1.b[2]\n"
       "udot z25.s, z6.b, z2.b[2]\n"
       "udot z29.s, z6.b, z3.b[2]\n"
@@ -1226,16 +1226,16 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z31.s, z8.b, z3.b[2]\n"
       "ble 52f\n"
       "ld1b { z9.b }, p2/Z, [x28]\n"
-      "udot z16.s, z9.b, z0.b[3]\n"
       "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "udot z16.s, z9.b, z0.b[3]\n"
       "udot z20.s, z9.b, z1.b[3]\n"
       "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "udot z24.s, z9.b, z2.b[3]\n"
       "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "udot z24.s, z9.b, z2.b[3]\n"
       "udot z28.s, z9.b, z3.b[3]\n"
       "udot z17.s, z10.b, z0.b[3]\n"
       "udot z21.s, z10.b, z1.b[3]\n"
+      "addvl x28, x28, #4\n"
       "udot z25.s, z10.b, z2.b[3]\n"
       "udot z29.s, z10.b, z3.b[3]\n"
       "udot z18.s, z4.b, z0.b[3]\n"
@@ -1253,27 +1253,27 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "udot z13.s, z2.b, z15.b\n"
       "udot z14.s, z3.b, z15.b\n"
       "53:"  // Height 4: Multiply loop: unique 8: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 46b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x22, x26, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x23, x27, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "tbnz %x[flags], #31, 54f\n"
-      "add x19, %x[qp], %[b_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
-      "neg z4.s, p2/M, z4.s\n"
-      "mov x19, #0x4\n"
-      "whilelt p0.s, XZR, x19\n"
+      "mov x20, #0x4\n"
+      "whilelt p0.s, XZR, x20\n"
+      "add x20, %x[qp], %[b_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       "uaddv d11, p0, z11.s\n"
+      "mov z11.s, z11.s[0]\n"
       "uaddv d12, p0, z12.s\n"
       "uaddv d13, p0, z13.s\n"
-      "uaddv d14, p0, z14.s\n"
-      "mov z11.s, z11.s[0]\n"
       "mov z12.s, z12.s[0]\n"
       "mov z13.s, z13.s[0]\n"
+      "uaddv d14, p0, z14.s\n"
+      "neg z4.s, p2/M, z4.s\n"
       "mov z14.s, z14.s[0]\n"
       "mul z11.s, p2/M, z11.s, z4.s\n"
       "mul z12.s, p2/M, z12.s, z4.s\n"
@@ -1281,24 +1281,24 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "mul z14.s, p2/M, z14.s, z4.s\n"
       "54:"  // Height 4: skip row sum fixup
       "add z16.s, z16.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x19, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z20.s, z20.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
       "add z21.s, z21.s, z12.s\n"
+      "add x20, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z22.s, z22.s, z12.s\n"
       "add z23.s, z23.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
+      "add x20, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z28.s, z28.s, z14.s\n"
@@ -1318,8 +1318,8 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       "add z28.s, z28.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z29.s, z29.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x20]\n"
       "add z30.s, z30.s, z2.s\n"
       "add z31.s, z31.s, z3.s\n"
       ".inst 0x04a47610  // sqrdmulh z16.s, z16.s, z4.s\n"
@@ -1341,160 +1341,160 @@ void sve_hybrid_u8qa_dot_4x4VL (
       "tbz %x[flags], #5, 55f\n"
       "and z4.d, z16.d, z0.d\n"
       "and z5.d, z17.d, z0.d\n"
-      "and z6.d, z18.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z16.s, z16.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
-      "sqadd z18.s, z18.s, z6.s\n"
+      "and z6.d, z18.d, z0.d\n"
       "and z7.d, z19.d, z0.d\n"
       "and z8.d, z20.d, z0.d\n"
       "and z9.d, z21.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
-      "sqadd z19.s, z19.s, z7.s\n"
-      "sqadd z20.s, z20.s, z8.s\n"
-      "sqadd z21.s, z21.s, z9.s\n"
       "and z10.d, z22.d, z0.d\n"
       "and z4.d, z23.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
+      "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z18.s, z18.s, z6.s\n"
+      "sqadd z19.s, z19.s, z7.s\n"
+      "sqadd z20.s, z20.s, z8.s\n"
+      "sqadd z21.s, z21.s, z9.s\n"
       "sqadd z22.s, z22.s, z10.s\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
-      "asr z6.s, z6.s, #0x1f\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "sqadd z25.s, z25.s, z6.s\n"
-      "sqadd z26.s, z26.s, z7.s\n"
-      "sqadd z27.s, z27.s, z8.s\n"
       "and z9.d, z28.d, z0.d\n"
       "and z10.d, z29.d, z0.d\n"
       "and z4.d, z30.d, z0.d\n"
+      "and z5.d, z31.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z25.s, z25.s, z6.s\n"
+      "sqadd z26.s, z26.s, z7.s\n"
+      "sqadd z27.s, z27.s, z8.s\n"
       "sqadd z28.s, z28.s, z9.s\n"
       "sqadd z29.s, z29.s, z10.s\n"
       "sqadd z30.s, z30.s, z4.s\n"
-      "and z5.d, z31.d, z0.d\n"
-      "asr z5.s, z5.s, #0x1f\n"
       "sqadd z31.s, z31.s, z5.s\n"
       "55:"  // Height 4: no shift correction
+      "add x20, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x20]\n"
       ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add x19, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x19]\n"
+      "add z16.s, z16.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x19, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x19]\n"
-      "add x19, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x19]\n"
-      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add z16.s, z16.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
       "add z19.s, z19.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
+      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      "add z21.s, z21.s, z4.s\n"
+      "add z22.s, z22.s, z4.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
+      "add x20, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x20]\n"
+      "add z31.s, z31.s, z4.s\n"
+      "add x20, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x20]\n"
       "smin z16.s, p2/M, z16.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
       "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z20.s, p2/M, z20.s, z6.s\n"
+      "smin z21.s, p2/M, z21.s, z6.s\n"
+      "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z31.s, p2/M, z31.s, z6.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
-      "smin z20.s, p2/M, z20.s, z6.s\n"
       "uzp1 z16.h, z16.h, z17.h\n"
-      ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "uzp1 z17.h, z18.h, z19.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
+      "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x26]\n"
-      "add z21.s, z21.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z21.s, p2/M, z21.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z22.s, z22.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smin z22.s, p2/M, z22.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z20.h, z20.h, z21.h\n"
       "smax z22.s, p2/M, z22.s, z5.s\n"
+      "uzp1 z20.h, z20.h, z21.h\n"
+      "st1b { z16.b }, p1, [x27]\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z21.h, z22.h, z23.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z20.b, z20.b, z21.b\n"
-      "st1b { z20.b }, p1, [x22]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
-      "add z28.s, z28.s, z4.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z20.b }, p1, [x23]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
-      "add z29.s, z29.s, z4.s\n"
-      "uzp1 z25.h, z26.h, z27.h\n"
       "smax z28.s, p2/M, z28.s, z5.s\n"
-      "add z30.s, z30.s, z4.s\n"
+      "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x21]\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
-      ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "uzp1 z28.h, z28.h, z29.h\n"
-      "smin z31.s, p2/M, z31.s, z6.s\n"
+      "st1b { z24.b }, p1, [x22]\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "uzp1 z29.h, z30.h, z31.h\n"
       "uzp1 z28.b, z28.b, z29.b\n"
-      "st1b { z28.b }, p1, [x20]\n"
+      "st1b { z28.b }, p1, [x21]\n"
+      "addvl x27, x27, #1\n"
       "56:"  // Height 4: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 44b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 58f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "57:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "58:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp
index 0f3f5e35e1..f9d38c2925 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -85,18 +85,18 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 29f\n"
       "beq 15f\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z15.b, #0x1\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x27, %x[col_bias]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -104,119 +104,119 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "3:"  // Height 1: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "4:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 5f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "cbnz x25, 6f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "cbnz x26, 6f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
       "b 6f\n"
       "5:"  // Height 1: setup direct input
-      "mov x23, %x[input_ptr]\n"
+      "mov x24, %x[input_ptr]\n"
       "6:"  // Height 1: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 9f\n"
       "7:"  // Height 1: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x23, x23, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
       ".inst 0x45c69837  // ummla z23.s, z1.b, z6.b\n"
+      "add x24, x24, #0x10\n"
       "tbnz %x[flags], #31, 8f\n"
       "udot z11.s, z0.b, z15.b\n"
       "udot z11.s, z1.b, z15.b\n"
       "8:"  // Height 1: Multiply loop: unique 1: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 7b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
+      "addvl x28, x28, #8\n"
       "ble 10f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
+      ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
-      ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
       ".inst 0x45c69837  // ummla z23.s, z1.b, z6.b\n"
+      "addvl x28, x28, #8\n"
       "10:"  // Height 1: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 11f\n"
       "udot z11.s, z0.b, z15.b\n"
       "udot z11.s, z1.b, z15.b\n"
       "11:"  // Height 1: Multiply loop: unique 2: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 4b\n"
       "uzp1 z16.d, z16.d, z20.d\n"
       "uzp1 z17.d, z17.d, z21.d\n"
@@ -224,33 +224,33 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "uzp1 z19.d, z19.d, z23.d\n"
       "mov z23.d, z16.d\n"
       "tbnz %x[flags], #31, 12f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z1.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z1.s }, p2/Z, [x22]\n"
-      "mov z11.s, z11.s[0]\n"
       "neg z1.s, p2/M, z1.s\n"
+      "mov z11.s, z11.s[0]\n"
       "mul z11.s, p2/M, z11.s, z1.s\n"
       "12:"  // Height 1: skip row sum fixup
       "add z23.s, z23.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z17.s, z17.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z18.s, z18.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z19.s, z19.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z23.s, z23.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
       ".inst 0x04a47631  // sqrdmulh z17.s, z17.s, z4.s\n"
+      "addvl x10, x10, #4\n"
       ".inst 0x04a47652  // sqrdmulh z18.s, z18.s, z4.s\n"
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 13f\n"
@@ -261,26 +261,26 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
       "sqadd z17.s, z17.s, z5.s\n"
       "sqadd z18.s, z18.s, z6.s\n"
-      "asr z7.s, z7.s, #0x1f\n"
       "sqadd z19.s, z19.s, z7.s\n"
       "13:"  // Height 1: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z17.s, z17.s, z4.s\n"
       "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
       "add z19.s, z19.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z17.s, p2/M, z17.s, z6.s\n"
       "smin z18.s, p2/M, z18.s, z6.s\n"
@@ -288,31 +288,31 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
-      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z23.h, z23.h, z17.h\n"
+      "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z23.b, z23.b, z17.b\n"
-      "st1b { z23.b }, p1, [x26]\n"
-      "addvl x26, x26, #1\n"
+      "st1b { z23.b }, p1, [x27]\n"
+      "addvl x27, x27, #1\n"
       "14:"  // Height 1: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 2b\n"
       "b 58f\n"
       "15:"  // Height 2
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z15.b, #0x1\n"
-      "mov x26, %x[output_ptr]\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "16:"  // Height 2: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -320,130 +320,130 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "17:"  // Height 2: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "18:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 19f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "cbnz x25, 20f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "cbnz x26, 20f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 20f\n"
       "19:"  // Height 2: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
       "20:"  // Height 2: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 23f\n"
       "21:"  // Height 2: Multiply loop: Main loop head
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x22, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
       ".inst 0x45c69837  // ummla z23.s, z1.b, z6.b\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "tbnz %x[flags], #31, 22f\n"
       "udot z11.s, z0.b, z15.b\n"
       "udot z11.s, z1.b, z15.b\n"
       "22:"  // Height 2: Multiply loop: unique 3: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 21b\n"
       "23:"  // Height 2: Multiply loop: Single iteration only
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "trn1 z0.d, z1.d, z2.d\n"
       "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
+      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
-      "trn1 z0.d, z1.d, z2.d\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "trn2 z1.d, z1.d, z2.d\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
+      "addvl x28, x28, #8\n"
       "ble 24f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
+      ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
-      ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
-      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
       ".inst 0x45c69837  // ummla z23.s, z1.b, z6.b\n"
+      "addvl x28, x28, #8\n"
       "24:"  // Height 2: Multiply loop: multiply skip
       "tbnz %x[flags], #31, 25f\n"
       "udot z11.s, z0.b, z15.b\n"
       "udot z11.s, z1.b, z15.b\n"
       "25:"  // Height 2: Multiply loop: unique 4: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 18b\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
+      "add x22, x27, x20\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
       "uzp1 z21.d, z18.d, z22.d\n"
@@ -452,39 +452,39 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "uzp2 z19.d, z19.d, z23.d\n"
       "mov z23.d, z7.d\n"
       "tbnz %x[flags], #31, 26f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z2.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z2.s }, p2/Z, [x22]\n"
+      "neg z2.s, p2/M, z2.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z2.s, p2/M, z2.s\n"
       "mul z11.s, p2/M, z11.s, z2.s\n"
       "mul z12.s, p2/M, z12.s, z2.s\n"
       "26:"  // Height 2: skip row sum fixup
       "add z23.s, z23.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z23.s, z23.s, z0.s\n"
       "add z20.s, z20.s, z1.s\n"
+      "addvl x10, x10, #4\n"
       "add z21.s, z21.s, z2.s\n"
       "add z22.s, z22.s, z3.s\n"
       "add z16.s, z16.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z17.s, z17.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       ".inst 0x04a476f7  // sqrdmulh z23.s, z23.s, z4.s\n"
@@ -497,97 +497,97 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       ".inst 0x04a47673  // sqrdmulh z19.s, z19.s, z4.s\n"
       "tbz %x[flags], #5, 27f\n"
       "and z4.d, z23.d, z0.d\n"
-      "and z5.d, z20.d, z0.d\n"
-      "and z6.d, z21.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z23.s, z23.s, z4.s\n"
-      "sqadd z20.s, z20.s, z5.s\n"
-      "sqadd z21.s, z21.s, z6.s\n"
+      "and z5.d, z20.d, z0.d\n"
+      "and z6.d, z21.d, z0.d\n"
       "and z7.d, z22.d, z0.d\n"
       "and z8.d, z16.d, z0.d\n"
       "and z9.d, z17.d, z0.d\n"
+      "and z10.d, z18.d, z0.d\n"
+      "and z4.d, z19.d, z0.d\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "sqadd z20.s, z20.s, z5.s\n"
+      "sqadd z21.s, z21.s, z6.s\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "sqadd z16.s, z16.s, z8.s\n"
       "sqadd z17.s, z17.s, z9.s\n"
-      "and z10.d, z18.d, z0.d\n"
-      "and z4.d, z19.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
       "sqadd z18.s, z18.s, z10.s\n"
       "sqadd z19.s, z19.s, z4.s\n"
       "27:"  // Height 2: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z23.s, z23.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z23.s, z23.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z23.s, p2/M, z23.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z23.h, z23.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z23.b, z23.b, z20.b\n"
-      "st1b { z23.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z23.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
+      "st1b { z16.b }, p1, [x22]\n"
+      "addvl x27, x27, #1\n"
       "28:"  // Height 2: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 16b\n"
       "b 58f\n"
       "29:"  // Height 3
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
       "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "mov x26, %x[output_ptr]\n"
       "mov z15.b, #0x1\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
       "30:"  // Height 3: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -603,74 +603,74 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "31:"  // Height 3: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "32:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 33f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "cbnz x25, 34f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "cbnz x26, 34f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 34f\n"
       "33:"  // Height 3: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "34:"  // Height 3: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 37f\n"
       "35:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
+      ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
-      ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
       ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
       ".inst 0x45c8985d  // ummla z29.s, z2.b, z8.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
       ".inst 0x45c9985a  // ummla z26.s, z2.b, z9.b\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
       ".inst 0x45ca985e  // ummla z30.s, z2.b, z10.b\n"
       "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
       ".inst 0x45c4985b  // ummla z27.s, z2.b, z4.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
       ".inst 0x45c5985f  // ummla z31.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
       ".inst 0x45c69878  // ummla z24.s, z3.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
@@ -694,36 +694,36 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "udot z11.s, z1.b, z15.b\n"
       "udot z13.s, z3.b, z15.b\n"
       "36:"  // Height 3: Multiply loop: unique 5: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 35b\n"
       "37:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
-      ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
-      ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
+      "subs x25, x25, #0x8\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
       "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
+      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
-      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
       ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
       ".inst 0x45c8985d  // ummla z29.s, z2.b, z8.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
       ".inst 0x45c9985a  // ummla z26.s, z2.b, z9.b\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
@@ -735,23 +735,23 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "ble 38f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45c69878  // ummla z24.s, z3.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x45c7987c  // ummla z28.s, z3.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45c89879  // ummla z25.s, z3.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
       ".inst 0x45c9987d  // ummla z29.s, z3.b, z9.b\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
       ".inst 0x45ca987a  // ummla z26.s, z3.b, z10.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
       ".inst 0x45c4987e  // ummla z30.s, z3.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
@@ -765,17 +765,17 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "udot z11.s, z1.b, z15.b\n"
       "udot z13.s, z3.b, z15.b\n"
       "39:"  // Height 3: Multiply loop: unique 6: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 32b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
       "uzp1 z20.d, z17.d, z21.d\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "add x20, x21, x19\n"
+      "add x21, x22, x20\n"
       "uzp1 z21.d, z18.d, z22.d\n"
       "uzp2 z18.d, z18.d, z22.d\n"
       "uzp1 z22.d, z19.d, z23.d\n"
@@ -786,37 +786,37 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "uzp1 z27.d, z27.d, z31.d\n"
       "mov z31.d, z7.d\n"
       "tbnz %x[flags], #31, 40f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z3.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z3.s }, p2/Z, [x22]\n"
       ".inst 0x4491a9ad  // addp z13.s, p2/m, z13.s, z13.s\n"
+      "neg z3.s, p2/M, z3.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z3.s, p2/M, z3.s\n"
-      "mov z13.s, z13.s[0]\n"
       "mul z11.s, p2/M, z11.s, z3.s\n"
+      "mov z13.s, z13.s[0]\n"
       "mul z12.s, p2/M, z12.s, z3.s\n"
       "mul z13.s, p2/M, z13.s, z3.s\n"
       "40:"  // Height 3: skip row sum fixup
       "add z31.s, z31.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z24.s, z24.s, z13.s\n"
       "add z25.s, z25.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z26.s, z26.s, z13.s\n"
       "add z27.s, z27.s, z13.s\n"
       "add z31.s, z31.s, z0.s\n"
@@ -828,8 +828,8 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "add z18.s, z18.s, z2.s\n"
       "add z19.s, z19.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -848,131 +848,131 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z20.d, z0.d\n"
       "and z6.d, z21.d, z0.d\n"
+      "and z7.d, z22.d, z0.d\n"
+      "and z8.d, z16.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
       "sqadd z21.s, z21.s, z6.s\n"
-      "and z7.d, z22.d, z0.d\n"
-      "and z8.d, z16.d, z0.d\n"
-      "and z9.d, z17.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
       "sqadd z22.s, z22.s, z7.s\n"
       "sqadd z16.s, z16.s, z8.s\n"
-      "sqadd z17.s, z17.s, z9.s\n"
+      "and z9.d, z17.d, z0.d\n"
       "and z10.d, z18.d, z0.d\n"
       "and z4.d, z19.d, z0.d\n"
       "and z5.d, z24.d, z0.d\n"
-      "asr z10.s, z10.s, #0x1f\n"
-      "asr z4.s, z4.s, #0x1f\n"
-      "asr z5.s, z5.s, #0x1f\n"
-      "sqadd z18.s, z18.s, z10.s\n"
-      "sqadd z19.s, z19.s, z4.s\n"
-      "sqadd z24.s, z24.s, z5.s\n"
       "and z6.d, z25.d, z0.d\n"
       "and z7.d, z26.d, z0.d\n"
       "and z8.d, z27.d, z0.d\n"
+      "asr z9.s, z9.s, #0x1f\n"
+      "asr z10.s, z10.s, #0x1f\n"
+      "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
       "asr z6.s, z6.s, #0x1f\n"
       "asr z7.s, z7.s, #0x1f\n"
       "asr z8.s, z8.s, #0x1f\n"
+      "sqadd z17.s, z17.s, z9.s\n"
+      "sqadd z18.s, z18.s, z10.s\n"
+      "sqadd z19.s, z19.s, z4.s\n"
+      "sqadd z24.s, z24.s, z5.s\n"
       "sqadd z25.s, z25.s, z6.s\n"
       "sqadd z26.s, z26.s, z7.s\n"
       "sqadd z27.s, z27.s, z8.s\n"
       "41:"  // Height 3: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z31.h, z31.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z31.b, z31.b, z20.b\n"
-      "st1b { z31.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z24.s, z24.s, z4.s\n"
-      "add z25.s, z25.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z31.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z26.s, z26.s, z4.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
+      "uzp1 z24.h, z24.h, z25.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x20]\n"
+      "st1b { z24.b }, p1, [x21]\n"
+      "addvl x27, x27, #1\n"
       "42:"  // Height 3: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 30b\n"
       "b 58f\n"
       "43:"  // Height 4
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x4\n"
+      "mov x10, %x[col_bias]\n"
       "mov z11.s, #0x0\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x27, %x[col_bias]\n"
       "mov z12.s, #0x0\n"
-      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "bic %x[flags], %x[flags], #0x80000000\n"
       "mov z13.s, #0x0\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x26, %x[output_ptr]\n"
+      "bic %x[flags], %x[flags], #0x80000000\n"
+      "ldr x9, [%x[args_ptr], %[offsetof_N]]\n"
       "mov z14.s, #0x0\n"
-      "mov x19, #0x4\n"
       "mov z15.b, #0x1\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x28, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x27, %x[output_ptr]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "44:"  // Height 4: Column loop
+      "mov x20, #0x0\n"
+      "whilelt p1.b, x20, x9\n"
       "mov z16.s, #0x0\n"
-      "mov x19, #0x0\n"
       "mov z17.s, #0x0\n"
-      "whilelt p1.b, x19, x9\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "mov z20.s, #0x0\n"
@@ -988,85 +988,85 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "45:"  // Height 4: setup done
-      "mov x25, #0x0\n"
+      "mov x26, #0x0\n"
       "46:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w24, [x20, x25, LSL #0x2]\n"
+      "ldr w25, [x20, x26, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 47f\n"
-      "ldr x20, [%x[input_ptr], x25, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x23, [x20, #0x0]\n"
-      "ldr x22, [x20, #0x8]\n"
-      "ldr x21, [x20, #0x10]\n"
-      "ldr x20, [x20, #0x18]\n"
-      "cbnz x25, 48f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x24, [x21, #0x0]\n"
+      "ldr x23, [x21, #0x8]\n"
+      "ldr x22, [x21, #0x10]\n"
+      "ldr x21, [x21, #0x18]\n"
+      "cbnz x26, 48f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 48f\n"
       "47:"  // Height 4: setup direct input
-      "mov x23, %x[input_ptr]\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x24, %x[input_ptr]\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "48:"  // Height 4: input setup done
-      "cmp x24, #0x10\n"
+      "cmp x25, #0x10\n"
       "ble 51f\n"
       "49:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "add x23, x23, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
-      "add x22, x22, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
+      "ld1rqb { z4.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x20]\n"
-      "add x21, x21, #0x10\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x20, x20, #0x10\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
+      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
+      ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
+      ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      ".inst 0x45c8985d  // ummla z29.s, z2.b, z8.b\n"
       "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       "addvl x28, x28, #16\n"
-      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n"
-      ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
-      ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
-      ".inst 0x45c8985d  // ummla z29.s, z2.b, z8.b\n"
-      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
       ".inst 0x45c9985a  // ummla z26.s, z2.b, z9.b\n"
-      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n"
+      "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n"
       ".inst 0x45ca985e  // ummla z30.s, z2.b, z10.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x45c49813  // ummla z19.s, z0.b, z4.b\n"
+      "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n"
+      "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n"
       ".inst 0x45c4985b  // ummla z27.s, z2.b, z4.b\n"
-      "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
       ".inst 0x45c59817  // ummla z23.s, z0.b, z5.b\n"
+      "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c5985f  // ummla z31.s, z2.b, z5.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45c69878  // ummla z24.s, z3.b, z6.b\n"
       "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x45c7987c  // ummla z28.s, z3.b, z7.b\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x45c89879  // ummla z25.s, z3.b, z8.b\n"
       ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
       ".inst 0x45c9987d  // ummla z29.s, z3.b, z9.b\n"
@@ -1084,38 +1084,38 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "udot z11.s, z1.b, z15.b\n"
       "udot z13.s, z3.b, z15.b\n"
       "50:"  // Height 4: Multiply loop: unique 7: skip row sum
-      "sub x24, x24, #0x10\n"
-      "cmp x24, #0x10\n"
+      "sub x25, x25, #0x10\n"
+      "cmp x25, #0x10\n"
       "bgt 49b\n"
       "51:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z5.b }, p2/Z, [x28]\n"
-      "whilelt p0.b, XZR, x24\n"
-      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
-      "subs x24, x24, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x23]\n"
-      "ld1rqb { z2.b }, p0/Z, [x22]\n"
+      "whilelt p0.b, XZR, x25\n"
+      "ld1rqb { z1.b }, p0/Z, [x24]\n"
+      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x21]\n"
+      "ld1rqb { z3.b }, p0/Z, [x22]\n"
+      "ld1rqb { z4.b }, p0/Z, [x21]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x20]\n"
-      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "trn1 z2.d, z3.d, z4.d\n"
+      "ld1b { z5.b }, p2/Z, [x28]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c59810  // ummla z16.s, z0.b, z5.b\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
+      "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n"
+      "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n"
+      "subs x25, x25, #0x8\n"
       ".inst 0x45c69814  // ummla z20.s, z0.b, z6.b\n"
       "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n"
-      "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
-      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
+      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       ".inst 0x45c79811  // ummla z17.s, z0.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
-      ".inst 0x45c6985c  // ummla z28.s, z2.b, z6.b\n"
       ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
       ".inst 0x45c89815  // ummla z21.s, z0.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n"
       ".inst 0x45c8985d  // ummla z29.s, z2.b, z8.b\n"
       ".inst 0x45c99812  // ummla z18.s, z0.b, z9.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45c9985a  // ummla z26.s, z2.b, z9.b\n"
       ".inst 0x45ca9816  // ummla z22.s, z0.b, z10.b\n"
       ".inst 0x45ca985e  // ummla z30.s, z2.b, z10.b\n"
@@ -1126,23 +1126,23 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "ble 52f\n"
       "ld1b { z6.b }, p2/Z, [x28]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
-      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       ".inst 0x45c69878  // ummla z24.s, z3.b, z6.b\n"
+      "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n"
       "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n"
       "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n"
       ".inst 0x45c79834  // ummla z20.s, z1.b, z7.b\n"
-      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       ".inst 0x45c7987c  // ummla z28.s, z3.b, z7.b\n"
+      "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n"
       "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n"
       ".inst 0x45c89831  // ummla z17.s, z1.b, z8.b\n"
-      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       ".inst 0x45c89879  // ummla z25.s, z3.b, z8.b\n"
+      "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n"
       "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n"
-      "addvl x28, x28, #8\n"
       ".inst 0x45c99835  // ummla z21.s, z1.b, z9.b\n"
       ".inst 0x45c9987d  // ummla z29.s, z3.b, z9.b\n"
       ".inst 0x45ca9832  // ummla z18.s, z1.b, z10.b\n"
       ".inst 0x45ca987a  // ummla z26.s, z3.b, z10.b\n"
+      "addvl x28, x28, #8\n"
       ".inst 0x45c49836  // ummla z22.s, z1.b, z4.b\n"
       ".inst 0x45c4987e  // ummla z30.s, z3.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
@@ -1156,19 +1156,19 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "udot z11.s, z1.b, z15.b\n"
       "udot z13.s, z3.b, z15.b\n"
       "53:"  // Height 4: Multiply loop: unique 8: skip row sum
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x25, x25, #0x1\n"
-      "cmp x25, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x26, x26, #0x1\n"
+      "cmp x26, x20\n"
       "bne 46b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp1 z7.d, z16.d, z20.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x27, x20\n"
+      "add x21, x22, x20\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "add x21, x26, x19\n"
       "uzp1 z20.d, z17.d, z21.d\n"
+      "add x20, x21, x20\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "add x20, x21, x19\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "add x19, x20, x19\n"
       "uzp2 z18.d, z18.d, z22.d\n"
       "uzp1 z22.d, z19.d, z23.d\n"
       "uzp2 z19.d, z19.d, z23.d\n"
@@ -1182,39 +1182,39 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "uzp2 z27.d, z27.d, z31.d\n"
       "mov z31.d, z7.d\n"
       "tbnz %x[flags], #31, 54f\n"
+      "add x23, %x[qp], %[b_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4491a96b  // addp z11.s, p2/m, z11.s, z11.s\n"
-      "add x22, %x[qp], %[b_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       ".inst 0x4491a9ad  // addp z13.s, p2/m, z13.s, z13.s\n"
+      "neg z4.s, p2/M, z4.s\n"
       "mov z12.s, z11.s[3]\n"
       "mov z11.s, z11.s[0]\n"
-      "neg z4.s, p2/M, z4.s\n"
+      "mul z11.s, p2/M, z11.s, z4.s\n"
       "mov z14.s, z13.s[3]\n"
       "mov z13.s, z13.s[0]\n"
-      "mul z11.s, p2/M, z11.s, z4.s\n"
       "mul z12.s, p2/M, z12.s, z4.s\n"
       "mul z13.s, p2/M, z13.s, z4.s\n"
       "mul z14.s, p2/M, z14.s, z4.s\n"
       "54:"  // Height 4: skip row sum fixup
       "add z31.s, z31.s, z11.s\n"
-      "ld1w { z0.s }, p2/Z, [x27]\n"
-      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z20.s, z20.s, z11.s\n"
-      "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n"
-      "add x23, %x[qp], %[per_layer_right_shift]\n"
+      "ld1w { z0.s }, p2/Z, [x10]\n"
+      "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
       "add z21.s, z21.s, z11.s\n"
-      "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n"
-      "add x22, %x[qp], %[per_layer_mul]\n"
       "add z22.s, z22.s, z11.s\n"
-      "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n"
-      "addvl x27, x27, #4\n"
+      "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+      "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
       "add z16.s, z16.s, z12.s\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
       "add z17.s, z17.s, z12.s\n"
+      "add x23, %x[qp], %[per_layer_mul]\n"
+      "orr %x[flags], %x[flags], #0x80000000\n"
       "add z18.s, z18.s, z12.s\n"
       "add z19.s, z19.s, z12.s\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
+      "add x23, %x[qp], %[per_layer_right_shift]\n"
       "add z23.s, z23.s, z13.s\n"
       "add z28.s, z28.s, z13.s\n"
+      "addvl x10, x10, #4\n"
       "add z29.s, z29.s, z13.s\n"
       "add z30.s, z30.s, z13.s\n"
       "add z24.s, z24.s, z14.s\n"
@@ -1234,8 +1234,8 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "add z29.s, z29.s, z2.s\n"
       "add z30.s, z30.s, z3.s\n"
       "add z24.s, z24.s, z0.s\n"
-      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z25.s, z25.s, z1.s\n"
+      "ld1rw { z0.s }, p2/Z, [x23]\n"
       "add z26.s, z26.s, z2.s\n"
       "add z27.s, z27.s, z3.s\n"
       ".inst 0x04a477ff  // sqrdmulh z31.s, z31.s, z4.s\n"
@@ -1257,160 +1257,160 @@ void sve_hybrid_u8qa_mmla_4x4VL (
       "tbz %x[flags], #5, 55f\n"
       "and z4.d, z31.d, z0.d\n"
       "and z5.d, z20.d, z0.d\n"
-      "and z6.d, z21.d, z0.d\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
-      "asr z6.s, z6.s, #0x1f\n"
       "sqadd z31.s, z31.s, z4.s\n"
       "sqadd z20.s, z20.s, z5.s\n"
-      "sqadd z21.s, z21.s, z6.s\n"
+      "and z6.d, z21.d, z0.d\n"
       "and z7.d, z22.d, z0.d\n"
       "and z8.d, z16.d, z0.d\n"
       "and z9.d, z17.d, z0.d\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "asr z9.s, z9.s, #0x1f\n"
-      "sqadd z22.s, z22.s, z7.s\n"
-      "sqadd z16.s, z16.s, z8.s\n"
-      "sqadd z17.s, z17.s, z9.s\n"
       "and z10.d, z18.d, z0.d\n"
       "and z4.d, z19.d, z0.d\n"
       "and z5.d, z23.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
+      "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
       "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z21.s, z21.s, z6.s\n"
+      "sqadd z22.s, z22.s, z7.s\n"
+      "sqadd z16.s, z16.s, z8.s\n"
+      "sqadd z17.s, z17.s, z9.s\n"
       "sqadd z18.s, z18.s, z10.s\n"
       "sqadd z19.s, z19.s, z4.s\n"
       "sqadd z23.s, z23.s, z5.s\n"
       "and z6.d, z28.d, z0.d\n"
       "and z7.d, z29.d, z0.d\n"
       "and z8.d, z30.d, z0.d\n"
-      "asr z6.s, z6.s, #0x1f\n"
-      "asr z7.s, z7.s, #0x1f\n"
-      "asr z8.s, z8.s, #0x1f\n"
-      "sqadd z28.s, z28.s, z6.s\n"
-      "sqadd z29.s, z29.s, z7.s\n"
-      "sqadd z30.s, z30.s, z8.s\n"
       "and z9.d, z24.d, z0.d\n"
       "and z10.d, z25.d, z0.d\n"
       "and z4.d, z26.d, z0.d\n"
+      "and z5.d, z27.d, z0.d\n"
+      "asr z6.s, z6.s, #0x1f\n"
+      "asr z7.s, z7.s, #0x1f\n"
+      "asr z8.s, z8.s, #0x1f\n"
       "asr z9.s, z9.s, #0x1f\n"
       "asr z10.s, z10.s, #0x1f\n"
       "asr z4.s, z4.s, #0x1f\n"
+      "asr z5.s, z5.s, #0x1f\n"
+      "sqadd z28.s, z28.s, z6.s\n"
+      "sqadd z29.s, z29.s, z7.s\n"
+      "sqadd z30.s, z30.s, z8.s\n"
       "sqadd z24.s, z24.s, z9.s\n"
       "sqadd z25.s, z25.s, z10.s\n"
       "sqadd z26.s, z26.s, z4.s\n"
-      "and z5.d, z27.d, z0.d\n"
-      "asr z5.s, z5.s, #0x1f\n"
       "sqadd z27.s, z27.s, z5.s\n"
       "55:"  // Height 4: no shift correction
+      "add x23, %x[qp], %[c_offset]\n"
+      "ld1rw { z4.s }, p2/Z, [x23]\n"
       ".inst 0x4482881f  // srshl z31.s, p2/M, z31.s, z0.s\n"
-      "add x22, %x[qp], %[c_offset]\n"
-      "ld1rw { z4.s }, p2/Z, [x22]\n"
+      "add z31.s, z31.s, z4.s\n"
       ".inst 0x44828814  // srshl z20.s, p2/M, z20.s, z0.s\n"
-      "add x22, %x[qp], %[minval]\n"
       ".inst 0x44828815  // srshl z21.s, p2/M, z21.s, z0.s\n"
-      "ld1rw { z5.s }, p2/Z, [x22]\n"
-      "add x22, %x[qp], %[maxval]\n"
-      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
-      "ld1rw { z6.s }, p2/Z, [x22]\n"
-      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
-      "add z31.s, z31.s, z4.s\n"
       "add z20.s, z20.s, z4.s\n"
       "add z21.s, z21.s, z4.s\n"
+      ".inst 0x44828816  // srshl z22.s, p2/M, z22.s, z0.s\n"
+      ".inst 0x44828810  // srshl z16.s, p2/M, z16.s, z0.s\n"
       "add z22.s, z22.s, z4.s\n"
       "add z16.s, z16.s, z4.s\n"
+      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
+      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
+      "add z17.s, z17.s, z4.s\n"
+      "add z18.s, z18.s, z4.s\n"
+      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
+      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
+      "add z19.s, z19.s, z4.s\n"
+      "add z23.s, z23.s, z4.s\n"
+      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
+      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
+      "add z28.s, z28.s, z4.s\n"
+      "add z29.s, z29.s, z4.s\n"
+      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
+      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
+      "add z30.s, z30.s, z4.s\n"
+      "add z24.s, z24.s, z4.s\n"
+      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
+      "add z25.s, z25.s, z4.s\n"
+      "add z26.s, z26.s, z4.s\n"
+      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
+      "add x23, %x[qp], %[maxval]\n"
+      "ld1rw { z6.s }, p2/Z, [x23]\n"
+      "add z27.s, z27.s, z4.s\n"
+      "add x23, %x[qp], %[minval]\n"
+      "ld1rw { z5.s }, p2/Z, [x23]\n"
       "smin z31.s, p2/M, z31.s, z6.s\n"
       "smin z20.s, p2/M, z20.s, z6.s\n"
       "smin z21.s, p2/M, z21.s, z6.s\n"
       "smin z22.s, p2/M, z22.s, z6.s\n"
+      "smin z16.s, p2/M, z16.s, z6.s\n"
+      "smin z17.s, p2/M, z17.s, z6.s\n"
+      "smin z18.s, p2/M, z18.s, z6.s\n"
+      "smin z19.s, p2/M, z19.s, z6.s\n"
+      "smin z23.s, p2/M, z23.s, z6.s\n"
+      "smin z28.s, p2/M, z28.s, z6.s\n"
+      "smin z29.s, p2/M, z29.s, z6.s\n"
+      "smin z30.s, p2/M, z30.s, z6.s\n"
+      "smin z24.s, p2/M, z24.s, z6.s\n"
+      "smin z25.s, p2/M, z25.s, z6.s\n"
+      "smin z26.s, p2/M, z26.s, z6.s\n"
+      "smin z27.s, p2/M, z27.s, z6.s\n"
       "smax z31.s, p2/M, z31.s, z5.s\n"
       "smax z20.s, p2/M, z20.s, z5.s\n"
       "smax z21.s, p2/M, z21.s, z5.s\n"
-      "smax z22.s, p2/M, z22.s, z5.s\n"
-      "smin z16.s, p2/M, z16.s, z6.s\n"
       "uzp1 z31.h, z31.h, z20.h\n"
-      ".inst 0x44828811  // srshl z17.s, p2/M, z17.s, z0.s\n"
-      "uzp1 z20.h, z21.h, z22.h\n"
+      "smax z22.s, p2/M, z22.s, z5.s\n"
       "smax z16.s, p2/M, z16.s, z5.s\n"
+      "uzp1 z20.h, z21.h, z22.h\n"
       "uzp1 z31.b, z31.b, z20.b\n"
-      "st1b { z31.b }, p1, [x26]\n"
-      "add z17.s, z17.s, z4.s\n"
-      "addvl x26, x26, #1\n"
-      ".inst 0x44828812  // srshl z18.s, p2/M, z18.s, z0.s\n"
-      ".inst 0x44828813  // srshl z19.s, p2/M, z19.s, z0.s\n"
-      ".inst 0x44828817  // srshl z23.s, p2/M, z23.s, z0.s\n"
-      "smin z17.s, p2/M, z17.s, z6.s\n"
-      ".inst 0x4482881c  // srshl z28.s, p2/M, z28.s, z0.s\n"
-      "add z18.s, z18.s, z4.s\n"
-      "add z19.s, z19.s, z4.s\n"
-      "add z23.s, z23.s, z4.s\n"
-      "add z28.s, z28.s, z4.s\n"
       "smax z17.s, p2/M, z17.s, z5.s\n"
-      "smin z18.s, p2/M, z18.s, z6.s\n"
-      "smin z19.s, p2/M, z19.s, z6.s\n"
-      "smin z23.s, p2/M, z23.s, z6.s\n"
-      "uzp1 z16.h, z16.h, z17.h\n"
       "smax z18.s, p2/M, z18.s, z5.s\n"
+      "uzp1 z16.h, z16.h, z17.h\n"
+      "st1b { z31.b }, p1, [x27]\n"
       "smax z19.s, p2/M, z19.s, z5.s\n"
       "smax z23.s, p2/M, z23.s, z5.s\n"
-      "smin z28.s, p2/M, z28.s, z6.s\n"
-      ".inst 0x4482881d  // srshl z29.s, p2/M, z29.s, z0.s\n"
       "uzp1 z17.h, z18.h, z19.h\n"
-      ".inst 0x4482881e  // srshl z30.s, p2/M, z30.s, z0.s\n"
       "uzp1 z16.b, z16.b, z17.b\n"
-      "st1b { z16.b }, p1, [x21]\n"
-      "add z29.s, z29.s, z4.s\n"
       "smax z28.s, p2/M, z28.s, z5.s\n"
-      "add z30.s, z30.s, z4.s\n"
-      ".inst 0x44828818  // srshl z24.s, p2/M, z24.s, z0.s\n"
-      "smin z29.s, p2/M, z29.s, z6.s\n"
-      "uzp1 z23.h, z23.h, z28.h\n"
-      "smin z30.s, p2/M, z30.s, z6.s\n"
-      "add z24.s, z24.s, z4.s\n"
       "smax z29.s, p2/M, z29.s, z5.s\n"
-      ".inst 0x44828819  // srshl z25.s, p2/M, z25.s, z0.s\n"
+      "uzp1 z23.h, z23.h, z28.h\n"
+      "st1b { z16.b }, p1, [x22]\n"
       "smax z30.s, p2/M, z30.s, z5.s\n"
-      "smin z24.s, p2/M, z24.s, z6.s\n"
-      ".inst 0x4482881a  // srshl z26.s, p2/M, z26.s, z0.s\n"
-      "add z25.s, z25.s, z4.s\n"
-      "uzp1 z28.h, z29.h, z30.h\n"
       "smax z24.s, p2/M, z24.s, z5.s\n"
-      "add z26.s, z26.s, z4.s\n"
+      "uzp1 z28.h, z29.h, z30.h\n"
       "uzp1 z23.b, z23.b, z28.b\n"
-      "st1b { z23.b }, p1, [x20]\n"
-      "smin z25.s, p2/M, z25.s, z6.s\n"
-      "smin z26.s, p2/M, z26.s, z6.s\n"
-      ".inst 0x4482881b  // srshl z27.s, p2/M, z27.s, z0.s\n"
       "smax z25.s, p2/M, z25.s, z5.s\n"
       "smax z26.s, p2/M, z26.s, z5.s\n"
-      "add z27.s, z27.s, z4.s\n"
       "uzp1 z24.h, z24.h, z25.h\n"
-      "smin z27.s, p2/M, z27.s, z6.s\n"
+      "st1b { z23.b }, p1, [x21]\n"
       "smax z27.s, p2/M, z27.s, z5.s\n"
       "uzp1 z25.h, z26.h, z27.h\n"
       "uzp1 z24.b, z24.b, z25.b\n"
-      "st1b { z24.b }, p1, [x19]\n"
+      "st1b { z24.b }, p1, [x20]\n"
+      "addvl x27, x27, #1\n"
       "56:"  // Height 4: Writeback done
       "decw x9, ALL, MUL #4\n"
       "cmp x9, XZR\n"
       "bgt 44b\n"
       "subs %x[M], %x[M], #0x4\n"
       "beq 58f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 57f\n"
-      "add x20, x20, #0x4\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x4\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "57:"  // Update direct input
-      "mov x19, #0x4\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x4\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "58:"  // Exit
 
       : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
-      : "cc", "memory", "p0", "p1", "p2", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp
index 11f9165a3f..a7dbef329e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,23 +87,23 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "cmp %x[M], #0x2\n"
       "bgt 21f\n"
       "beq 11f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
       "b 4f\n"
       "3:"  // Height 1: no accumulate
       "mov z8.s, #0x0\n"
@@ -111,87 +111,87 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x25, x25, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x26, x26, #0x4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z11.s, z7.b, z0.b\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z11.s, z7.b, z0.b\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "bne 5b\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "10:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 62f\n"
       "11:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "12:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 13f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
       "b 14f\n"
       "13:"  // Height 2: no accumulate
       "mov z8.s, #0x0\n"
@@ -203,112 +203,112 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "14:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "15:"  // Height 2: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 16f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 17f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 17f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 17f\n"
       "16:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "17:"  // Height 2: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 19f\n"
       "18:"  // Height 2: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x25, x25, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x26, x26, #0x4\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "subs x26, x26, #0x4\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "subs x27, x27, #0x4\n"
+      "add x25, x25, #0x4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z11.s, z7.b, z0.b\n"
       "udot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 18b\n"
       "19:"  // Height 2: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
-      "addvl x9, x9, #4\n"
+      "addvl x10, x10, #4\n"
       "udot z11.s, z7.b, z0.b\n"
       "udot z15.s, z7.b, z1.b\n"
       "bne 15b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
       "20:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 12b\n"
       "b 62f\n"
       "21:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "22:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 23f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
       "b 24f\n"
       "23:"  // Height 3: no accumulate
       "mov z8.s, #0x0\n"
@@ -324,74 +324,74 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "24:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "25:"  // Height 3: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 26f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 27f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 27f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 27f\n"
       "26:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "27:"  // Height 3: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 29f\n"
       "28:"  // Height 3: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
-      "add x23, x23, #0x4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
+      "add x24, x24, #0x4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
       "udot z11.s, z7.b, z0.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "udot z15.s, z7.b, z1.b\n"
       "udot z19.s, z7.b, z2.b\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 28b\n"
       "29:"  // Height 3: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
@@ -399,61 +399,61 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "udot z15.s, z7.b, z1.b\n"
       "udot z19.s, z7.b, z2.b\n"
       "bne 25b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
       "30:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 22b\n"
       "b 62f\n"
       "31:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "32:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 33f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
       "b 34f\n"
       "33:"  // Height 4: no accumulate
       "mov z8.s, #0x0\n"
@@ -473,86 +473,86 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "34:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "35:"  // Height 4: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 36f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 37f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 37f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 37f\n"
       "36:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "37:"  // Height 4: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 39f\n"
       "38:"  // Height 4: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x24, x24, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x4\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
+      "add x24, x24, #0x4\n"
       "add x23, x23, #0x4\n"
-      "add x22, x22, #0x4\n"
       "udot z17.s, z7.b, z2.b\n"
       "udot z21.s, z7.b, z3.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
       "udot z22.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "udot z11.s, z7.b, z0.b\n"
       "udot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "udot z19.s, z7.b, z2.b\n"
       "udot z23.s, z7.b, z3.b\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 38b\n"
       "39:"  // Height 4: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "cmp x27, x19\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "cmp x28, x20\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
       "udot z21.s, z7.b, z3.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
@@ -562,71 +562,71 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "udot z19.s, z7.b, z2.b\n"
       "udot z23.s, z7.b, z3.b\n"
       "bne 35b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
       "40:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 32b\n"
       "b 62f\n"
       "41:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "42:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 43f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x20]\n"
-      "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x21]\n"
+      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
       "b 44f\n"
       "43:"  // Height 5: no accumulate
       "mov z8.s, #0x0\n"
@@ -650,98 +650,98 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "44:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "45:"  // Height 5: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 46f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 47f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 47f\n"
       "46:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "47:"  // Height 5: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 49f\n"
       "48:"  // Height 5: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "udot z24.s, z6.b, z4.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
-      "add x21, x21, #0x4\n"
+      "add x22, x22, #0x4\n"
       "udot z21.s, z7.b, z3.b\n"
       "udot z25.s, z7.b, z4.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
       "udot z22.s, z6.b, z3.b\n"
       "udot z26.s, z6.b, z4.b\n"
       "udot z11.s, z7.b, z0.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "udot z15.s, z7.b, z1.b\n"
       "udot z19.s, z7.b, z2.b\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
       "udot z23.s, z7.b, z3.b\n"
       "udot z27.s, z7.b, z4.b\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 48b\n"
       "49:"  // Height 5: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "udot z24.s, z6.b, z4.b\n"
       "udot z9.s, z7.b, z0.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
       "udot z21.s, z7.b, z3.b\n"
       "udot z25.s, z7.b, z4.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
@@ -753,84 +753,84 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "udot z23.s, z7.b, z3.b\n"
       "udot z27.s, z7.b, z4.b\n"
       "bne 45b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x20]\n"
-      "st1w { z25.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x21]\n"
+      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
       "50:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 42b\n"
       "b 62f\n"
       "51:"  // Height 6
-      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "mov x19, #0x18\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "madd %x[output_ptr], x20, x19, %x[output_ptr]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "mov x20, #0x18\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "52:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p0.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p0.s, x20, x11\n"
       "tbz %x[flags], #0, 53f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z8.s }, p3/Z, [x28]\n"
-      "ld1w { z9.s }, p2/Z, [x28, #1, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z10.s }, p1/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p0/Z, [x28, #3, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "ld1w { z12.s }, p3/Z, [x23]\n"
-      "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p3/Z, [x22]\n"
-      "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p3/Z, [x21]\n"
-      "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p3/Z, [x20]\n"
-      "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n"
-      "ld1w { z28.s }, p3/Z, [x19]\n"
-      "ld1w { z29.s }, p2/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p1/Z, [x19, #2, MUL VL]\n"
-      "ld1w { z31.s }, p0/Z, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p3/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p3/Z, [x24]\n"
+      "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p3/Z, [x23]\n"
+      "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p3/Z, [x22]\n"
+      "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p3/Z, [x21]\n"
+      "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p3/Z, [x20]\n"
+      "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n"
       "b 54f\n"
       "53:"  // Height 6: no accumulate
       "mov z8.s, #0x0\n"
@@ -858,110 +858,110 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "54:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "55:"  // Height 6: String loop
-      "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr w26, [x19, x27, LSL #0x2]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 56f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 57f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 57f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 57f\n"
       "56:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "57:"  // Height 6: input setup done
-      "subs x26, x26, #0x4\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "ble 59f\n"
       "58:"  // Height 6: Multiply loop: Main loop
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x25, x25, #0x4\n"
-      "subs x26, x26, #0x4\n"
+      "add x26, x26, #0x4\n"
+      "subs x27, x27, #0x4\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
+      "add x25, x25, #0x4\n"
       "add x24, x24, #0x4\n"
-      "add x23, x23, #0x4\n"
       "udot z24.s, z6.b, z4.b\n"
       "udot z28.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
-      "add x22, x22, #0x4\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
+      "add x23, x23, #0x4\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
+      "add x22, x22, #0x4\n"
       "add x21, x21, #0x4\n"
-      "add x20, x20, #0x4\n"
       "udot z17.s, z7.b, z2.b\n"
       "udot z21.s, z7.b, z3.b\n"
       "udot z25.s, z7.b, z4.b\n"
       "udot z29.s, z7.b, z5.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
       "udot z22.s, z6.b, z3.b\n"
       "udot z26.s, z6.b, z4.b\n"
       "udot z30.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9]\n"
+      "ld1b { z6.b }, p4/Z, [x10]\n"
       "udot z11.s, z7.b, z0.b\n"
       "udot z15.s, z7.b, z1.b\n"
-      "ld1rw { z0.s }, p4/Z, [x25]\n"
-      "ld1rw { z1.s }, p4/Z, [x24]\n"
+      "ld1rw { z0.s }, p4/Z, [x26]\n"
+      "ld1rw { z1.s }, p4/Z, [x25]\n"
       "udot z19.s, z7.b, z2.b\n"
       "udot z23.s, z7.b, z3.b\n"
-      "ld1rw { z2.s }, p4/Z, [x23]\n"
-      "ld1rw { z3.s }, p4/Z, [x22]\n"
+      "ld1rw { z2.s }, p4/Z, [x24]\n"
+      "ld1rw { z3.s }, p4/Z, [x23]\n"
       "udot z27.s, z7.b, z4.b\n"
       "udot z31.s, z7.b, z5.b\n"
-      "ld1rw { z4.s }, p4/Z, [x21]\n"
-      "ld1rw { z5.s }, p4/Z, [x20]\n"
-      "ld1b { z7.b }, p4/Z, [x9, #1, MUL VL]\n"
+      "ld1rw { z4.s }, p4/Z, [x22]\n"
+      "ld1rw { z5.s }, p4/Z, [x21]\n"
+      "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n"
       "bgt 58b\n"
       "59:"  // Height 6: Multiply loop: Main loop skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
       "udot z8.s, z6.b, z0.b\n"
       "udot z12.s, z6.b, z1.b\n"
-      "add x27, x27, #0x1\n"
+      "add x28, x28, #0x1\n"
       "udot z16.s, z6.b, z2.b\n"
       "udot z20.s, z6.b, z3.b\n"
-      "cmp x27, x19\n"
+      "cmp x28, x20\n"
       "udot z24.s, z6.b, z4.b\n"
       "udot z28.s, z6.b, z5.b\n"
-      "ld1b { z6.b }, p4/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b\n"
       "udot z13.s, z7.b, z1.b\n"
       "udot z17.s, z7.b, z2.b\n"
       "udot z21.s, z7.b, z3.b\n"
       "udot z25.s, z7.b, z4.b\n"
       "udot z29.s, z7.b, z5.b\n"
-      "ld1b { z7.b }, p4/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b\n"
       "udot z14.s, z6.b, z1.b\n"
       "udot z18.s, z6.b, z2.b\n"
@@ -975,57 +975,57 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx (
       "udot z27.s, z7.b, z4.b\n"
       "udot z31.s, z7.b, z5.b\n"
       "bne 55b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z8.s }, p3, [x28]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z9.s }, p2, [x28, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "st1w { z10.s }, p1, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p0, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p3, [x23]\n"
-      "st1w { z13.s }, p2, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p1, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p0, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p3, [x22]\n"
-      "st1w { z17.s }, p2, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p1, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p0, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p3, [x21]\n"
-      "st1w { z21.s }, p2, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p1, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p0, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p3, [x20]\n"
-      "st1w { z25.s }, p2, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p1, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p0, [x20, #3, MUL VL]\n"
-      "st1w { z28.s }, p3, [x19]\n"
-      "st1w { z29.s }, p2, [x19, #1, MUL VL]\n"
-      "st1w { z30.s }, p1, [x19, #2, MUL VL]\n"
-      "st1w { z31.s }, p0, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p3, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p2, [x9, #1, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "st1w { z10.s }, p1, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p0, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p3, [x24]\n"
+      "st1w { z13.s }, p2, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p0, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p3, [x23]\n"
+      "st1w { z17.s }, p2, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p1, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p0, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p3, [x22]\n"
+      "st1w { z21.s }, p2, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p1, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p0, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p3, [x21]\n"
+      "st1w { z25.s }, p2, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p1, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p0, [x21, #3, MUL VL]\n"
+      "st1w { z28.s }, p3, [x20]\n"
+      "st1w { z29.s }, p2, [x20, #1, MUL VL]\n"
+      "st1w { z30.s }, p1, [x20, #2, MUL VL]\n"
+      "st1w { z31.s }, p0, [x20, #3, MUL VL]\n"
       "60:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 52b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 62f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "61:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "62:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp
index fc8bdb50a9..30a108af7e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,23 +87,23 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 23f\n"
       "beq 12f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
       "b 4f\n"
       "3:"  // Height 1: no accumulate
       "mov z8.s, #0x0\n"
@@ -111,148 +111,148 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "cmp x26, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z11.s, z7.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z11.s, z7.b, z0.b[3]\n"
+      "add x26, x26, #0x10\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z9.s, z7.b, z0.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
-      "addvl x9, x9, #4\n"
       "udot z11.s, z7.b, z0.b[1]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z9.s, z7.b, z0.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
-      "addvl x9, x9, #4\n"
       "udot z11.s, z7.b, z0.b[2]\n"
+      "addvl x10, x10, #4\n"
       "ble 10f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z11.s, z7.b, z0.b[3]\n"
+      "addvl x10, x10, #4\n"
       "10:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 5b\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "11:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 68f\n"
       "12:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "13:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
       "b 15f\n"
       "14:"  // Height 2: no accumulate
       "mov z8.s, #0x0\n"
@@ -264,197 +264,197 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "15:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 18f\n"
       "17:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "18:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 20f\n"
       "19:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
       "udot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "udot z11.s, z7.b, z0.b[0]\n"
       "udot z15.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       "udot z8.s, z6.b, z0.b[1]\n"
       "udot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
       "udot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z11.s, z7.b, z0.b[3]\n"
       "udot z15.s, z7.b, z1.b[3]\n"
       "bgt 19b\n"
       "20:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
       "udot z13.s, z7.b, z1.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
+      "addvl x10, x10, #4\n"
       "udot z11.s, z7.b, z0.b[0]\n"
       "udot z15.s, z7.b, z1.b[0]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
+      "addvl x10, x10, #4\n"
       "udot z11.s, z7.b, z0.b[1]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
+      "addvl x10, x10, #4\n"
       "udot z11.s, z7.b, z0.b[2]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
       "ble 21f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
+      "addvl x10, x10, #4\n"
       "udot z11.s, z7.b, z0.b[3]\n"
       "udot z15.s, z7.b, z1.b[3]\n"
       "21:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
       "22:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 13b\n"
       "b 68f\n"
       "23:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "24:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
       "b 26f\n"
       "25:"  // Height 3: no accumulate
       "mov z8.s, #0x0\n"
@@ -470,99 +470,99 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
       "26:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "27:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 28f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 29f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 29f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 29f\n"
       "28:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "29:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 31f\n"
       "30:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "add x24, x24, #0x10\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "cmp x27, #0x10\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "udot z18.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       "udot z15.s, z7.b, z1.b[0]\n"
       "udot z19.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
       "udot z19.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
       "udot z19.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -571,22 +571,22 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z19.s, z7.b, z2.b[3]\n"
       "bgt 30b\n"
       "31:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z17.s, z7.b, z2.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
@@ -594,18 +594,18 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z15.s, z7.b, z1.b[0]\n"
       "udot z19.s, z7.b, z2.b[0]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
@@ -613,18 +613,18 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z15.s, z7.b, z1.b[1]\n"
       "udot z19.s, z7.b, z2.b[1]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
@@ -632,17 +632,17 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z15.s, z7.b, z1.b[2]\n"
       "udot z19.s, z7.b, z2.b[2]\n"
       "ble 32f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -650,65 +650,65 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z15.s, z7.b, z1.b[3]\n"
       "udot z19.s, z7.b, z2.b[3]\n"
       "32:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 27b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "33:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 24b\n"
       "b 68f\n"
       "34:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "35:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "b 37f\n"
       "36:"  // Height 4: no accumulate
       "mov z8.s, #0x0\n"
@@ -728,118 +728,118 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "37:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "38:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 39f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 40f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 40f\n"
       "39:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "40:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 42f\n"
       "41:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
       "udot z16.s, z6.b, z2.b[0]\n"
-      "add x23, x23, #0x10\n"
+      "udot z20.s, z6.b, z3.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x25, x25, #0x10\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
       "udot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
+      "add x24, x24, #0x10\n"
+      "add x23, x23, #0x10\n"
       "udot z17.s, z7.b, z2.b[0]\n"
-      "udot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
       "udot z22.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
       "udot z15.s, z7.b, z1.b[0]\n"
       "udot z19.s, z7.b, z2.b[0]\n"
       "udot z23.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
       "udot z22.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
       "udot z19.s, z7.b, z2.b[1]\n"
       "udot z23.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
       "udot z22.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
       "udot z19.s, z7.b, z2.b[2]\n"
       "udot z23.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -850,25 +850,25 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z23.s, z7.b, z3.b[3]\n"
       "bgt 41b\n"
       "42:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
       "udot z20.s, z6.b, z3.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z17.s, z7.b, z2.b[0]\n"
       "udot z21.s, z7.b, z3.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
@@ -878,20 +878,20 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z19.s, z7.b, z2.b[0]\n"
       "udot z23.s, z7.b, z3.b[0]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
@@ -901,20 +901,20 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z19.s, z7.b, z2.b[1]\n"
       "udot z23.s, z7.b, z3.b[1]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x4\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
@@ -924,19 +924,19 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z19.s, z7.b, z2.b[2]\n"
       "udot z23.s, z7.b, z3.b[2]\n"
       "ble 43f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -946,75 +946,75 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z19.s, z7.b, z2.b[3]\n"
       "udot z23.s, z7.b, z3.b[3]\n"
       "43:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 38b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
       "44:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 35b\n"
       "b 68f\n"
       "45:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "46:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x20]\n"
-      "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x21]\n"
+      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
       "b 48f\n"
       "47:"  // Height 5: no accumulate
       "mov z8.s, #0x0\n"
@@ -1038,137 +1038,137 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z26.s, #0x0\n"
       "mov z27.s, #0x0\n"
       "48:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 51f\n"
       "50:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "51:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 53f\n"
       "52:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "add x23, x23, #0x10\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
-      "add x22, x22, #0x10\n"
-      "udot z17.s, z7.b, z2.b[0]\n"
-      "add x21, x21, #0x10\n"
       "udot z20.s, z6.b, z3.b[0]\n"
+      "add x25, x25, #0x10\n"
       "udot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x24, x24, #0x10\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
+      "udot z17.s, z7.b, z2.b[0]\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       "udot z21.s, z7.b, z3.b[0]\n"
       "udot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
       "udot z22.s, z6.b, z3.b[0]\n"
       "udot z26.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "udot z15.s, z7.b, z1.b[0]\n"
       "udot z19.s, z7.b, z2.b[0]\n"
       "udot z23.s, z7.b, z3.b[0]\n"
       "udot z27.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
       "udot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
       "udot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
       "udot z22.s, z6.b, z3.b[1]\n"
       "udot z26.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
       "udot z19.s, z7.b, z2.b[1]\n"
       "udot z23.s, z7.b, z3.b[1]\n"
       "udot z27.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
       "udot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
       "udot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
       "udot z22.s, z6.b, z3.b[2]\n"
       "udot z26.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
       "udot z19.s, z7.b, z2.b[2]\n"
       "udot z23.s, z7.b, z3.b[2]\n"
       "udot z27.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
       "udot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
       "udot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -1181,28 +1181,28 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z27.s, z7.b, z4.b[3]\n"
       "bgt 52b\n"
       "53:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
       "udot z20.s, z6.b, z3.b[0]\n"
       "udot z24.s, z6.b, z4.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z17.s, z7.b, z2.b[0]\n"
       "udot z21.s, z7.b, z3.b[0]\n"
       "udot z25.s, z7.b, z4.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
@@ -1214,22 +1214,22 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z23.s, z7.b, z3.b[0]\n"
       "udot z27.s, z7.b, z4.b[0]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "udot z24.s, z6.b, z4.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
       "udot z25.s, z7.b, z4.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
@@ -1241,22 +1241,22 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z23.s, z7.b, z3.b[1]\n"
       "udot z27.s, z7.b, z4.b[1]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "udot z24.s, z6.b, z4.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
       "udot z25.s, z7.b, z4.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
@@ -1268,21 +1268,21 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z23.s, z7.b, z3.b[2]\n"
       "udot z27.s, z7.b, z4.b[2]\n"
       "ble 54f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
       "udot z24.s, z6.b, z4.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
       "udot z25.s, z7.b, z4.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -1294,88 +1294,88 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z23.s, z7.b, z3.b[3]\n"
       "udot z27.s, z7.b, z4.b[3]\n"
       "54:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "55:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 46b\n"
       "b 68f\n"
       "56:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "57:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z8.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z9.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z10.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z11.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
-      "ld1w { z16.s }, p4/Z, [x22]\n"
-      "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n"
-      "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n"
-      "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
-      "ld1w { z24.s }, p4/Z, [x20]\n"
-      "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n"
-      "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n"
-      "ld1w { z28.s }, p4/Z, [x19]\n"
-      "ld1w { z29.s }, p3/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x19, #2, MUL VL]\n"
-      "ld1w { z31.s }, p1/Z, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z8.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z16.s }, p4/Z, [x23]\n"
+      "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n"
+      "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p4/Z, [x21]\n"
+      "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x20]\n"
+      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
       "b 59f\n"
       "58:"  // Height 6: no accumulate
       "mov z8.s, #0x0\n"
@@ -1403,156 +1403,156 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "59:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "60:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 62f\n"
       "61:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "62:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 64f\n"
       "63:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
-      "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "cmp x26, #0x10\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "sub x27, x27, #0x10\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
       "add x25, x25, #0x10\n"
-      "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "add x24, x24, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
+      "udot z8.s, z6.b, z0.b[0]\n"
+      "udot z12.s, z6.b, z1.b[0]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
+      "udot z20.s, z6.b, z3.b[0]\n"
       "add x23, x23, #0x10\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "add x22, x22, #0x10\n"
-      "udot z20.s, z6.b, z3.b[0]\n"
-      "add x21, x21, #0x10\n"
-      "udot z17.s, z7.b, z2.b[0]\n"
-      "add x20, x20, #0x10\n"
       "udot z24.s, z6.b, z4.b[0]\n"
       "udot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "add x21, x21, #0x10\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
+      "udot z17.s, z7.b, z2.b[0]\n"
       "udot z21.s, z7.b, z3.b[0]\n"
       "udot z25.s, z7.b, z4.b[0]\n"
       "udot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
       "udot z22.s, z6.b, z3.b[0]\n"
       "udot z26.s, z6.b, z4.b[0]\n"
       "udot z30.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[0]\n"
       "udot z15.s, z7.b, z1.b[0]\n"
       "udot z19.s, z7.b, z2.b[0]\n"
       "udot z23.s, z7.b, z3.b[0]\n"
       "udot z27.s, z7.b, z4.b[0]\n"
       "udot z31.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
       "udot z24.s, z6.b, z4.b[1]\n"
       "udot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
       "udot z25.s, z7.b, z4.b[1]\n"
       "udot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
       "udot z22.s, z6.b, z3.b[1]\n"
       "udot z26.s, z6.b, z4.b[1]\n"
       "udot z30.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
       "udot z15.s, z7.b, z1.b[1]\n"
       "udot z19.s, z7.b, z2.b[1]\n"
       "udot z23.s, z7.b, z3.b[1]\n"
       "udot z27.s, z7.b, z4.b[1]\n"
       "udot z31.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
       "udot z24.s, z6.b, z4.b[2]\n"
       "udot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
       "udot z25.s, z7.b, z4.b[2]\n"
       "udot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
       "udot z22.s, z6.b, z3.b[2]\n"
       "udot z26.s, z6.b, z4.b[2]\n"
       "udot z30.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[2]\n"
       "udot z15.s, z7.b, z1.b[2]\n"
       "udot z19.s, z7.b, z2.b[2]\n"
       "udot z23.s, z7.b, z3.b[2]\n"
       "udot z27.s, z7.b, z4.b[2]\n"
       "udot z31.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
       "udot z24.s, z6.b, z4.b[3]\n"
       "udot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
       "udot z25.s, z7.b, z4.b[3]\n"
       "udot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -1567,31 +1567,31 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z31.s, z7.b, z5.b[3]\n"
       "bgt 63b\n"
       "64:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z6.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
-      "ld1rqb { z0.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z0.b }, p0/Z, [x26]\n"
+      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "subs x27, x27, #0x4\n"
+      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z4.b }, p0/Z, [x22]\n"
+      "ld1rqb { z5.b }, p0/Z, [x21]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[0]\n"
-      "ld1rqb { z1.b }, p0/Z, [x24]\n"
-      "udot z9.s, z7.b, z0.b[0]\n"
-      "ld1rqb { z2.b }, p0/Z, [x23]\n"
-      "ld1rqb { z3.b }, p0/Z, [x22]\n"
       "udot z12.s, z6.b, z1.b[0]\n"
-      "ld1rqb { z4.b }, p0/Z, [x21]\n"
-      "udot z13.s, z7.b, z1.b[0]\n"
-      "ld1rqb { z5.b }, p0/Z, [x20]\n"
       "udot z16.s, z6.b, z2.b[0]\n"
       "udot z20.s, z6.b, z3.b[0]\n"
       "udot z24.s, z6.b, z4.b[0]\n"
       "udot z28.s, z6.b, z5.b[0]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "udot z9.s, z7.b, z0.b[0]\n"
+      "udot z13.s, z7.b, z1.b[0]\n"
       "udot z17.s, z7.b, z2.b[0]\n"
       "udot z21.s, z7.b, z3.b[0]\n"
       "udot z25.s, z7.b, z4.b[0]\n"
       "udot z29.s, z7.b, z5.b[0]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z14.s, z6.b, z1.b[0]\n"
       "udot z18.s, z6.b, z2.b[0]\n"
@@ -1605,24 +1605,24 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z27.s, z7.b, z4.b[0]\n"
       "udot z31.s, z7.b, z5.b[0]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[1]\n"
       "udot z16.s, z6.b, z2.b[1]\n"
       "udot z20.s, z6.b, z3.b[1]\n"
+      "subs x27, x27, #0x4\n"
       "udot z24.s, z6.b, z4.b[1]\n"
       "udot z28.s, z6.b, z5.b[1]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[1]\n"
       "udot z13.s, z7.b, z1.b[1]\n"
       "udot z17.s, z7.b, z2.b[1]\n"
       "udot z21.s, z7.b, z3.b[1]\n"
       "udot z25.s, z7.b, z4.b[1]\n"
       "udot z29.s, z7.b, z5.b[1]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[1]\n"
       "udot z14.s, z6.b, z1.b[1]\n"
       "udot z18.s, z6.b, z2.b[1]\n"
@@ -1636,24 +1636,24 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z27.s, z7.b, z4.b[1]\n"
       "udot z31.s, z7.b, z5.b[1]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x4\n"
       "udot z12.s, z6.b, z1.b[2]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
       "udot z20.s, z6.b, z3.b[2]\n"
+      "subs x27, x27, #0x4\n"
       "udot z24.s, z6.b, z4.b[2]\n"
       "udot z28.s, z6.b, z5.b[2]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[2]\n"
       "udot z13.s, z7.b, z1.b[2]\n"
       "udot z17.s, z7.b, z2.b[2]\n"
       "udot z21.s, z7.b, z3.b[2]\n"
       "udot z25.s, z7.b, z4.b[2]\n"
       "udot z29.s, z7.b, z5.b[2]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[2]\n"
       "udot z14.s, z6.b, z1.b[2]\n"
       "udot z18.s, z6.b, z2.b[2]\n"
@@ -1667,23 +1667,23 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z27.s, z7.b, z4.b[2]\n"
       "udot z31.s, z7.b, z5.b[2]\n"
       "ble 65f\n"
-      "ld1b { z6.b }, p5/Z, [x9]\n"
+      "ld1b { z6.b }, p5/Z, [x10]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n"
       "udot z8.s, z6.b, z0.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #1, MUL VL]\n"
       "udot z12.s, z6.b, z1.b[3]\n"
       "udot z16.s, z6.b, z2.b[3]\n"
       "udot z20.s, z6.b, z3.b[3]\n"
       "udot z24.s, z6.b, z4.b[3]\n"
       "udot z28.s, z6.b, z5.b[3]\n"
-      "ld1b { z6.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n"
       "udot z9.s, z7.b, z0.b[3]\n"
       "udot z13.s, z7.b, z1.b[3]\n"
       "udot z17.s, z7.b, z2.b[3]\n"
       "udot z21.s, z7.b, z3.b[3]\n"
       "udot z25.s, z7.b, z4.b[3]\n"
       "udot z29.s, z7.b, z5.b[3]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #3, MUL VL]\n"
-      "addvl x9, x9, #4\n"
+      "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n"
+      "addvl x10, x10, #4\n"
       "udot z10.s, z6.b, z0.b[3]\n"
       "udot z14.s, z6.b, z1.b[3]\n"
       "udot z18.s, z6.b, z2.b[3]\n"
@@ -1697,61 +1697,61 @@ void sve_hybrid_u8u32_dot_6x4VL (
       "udot z27.s, z7.b, z4.b[3]\n"
       "udot z31.s, z7.b, z5.b[3]\n"
       "65:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 60b\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "st1w { z8.s }, p4, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "st1w { z12.s }, p4, [x23]\n"
-      "add x20, x21, x19, LSL #2\n"
-      "st1w { z13.s }, p3, [x23, #1, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
-      "st1w { z14.s }, p2, [x23, #2, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z15.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z20.s }, p4, [x21]\n"
-      "st1w { z21.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z22.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z23.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
-      "st1w { z28.s }, p4, [x19]\n"
-      "st1w { z29.s }, p3, [x19, #1, MUL VL]\n"
-      "st1w { z30.s }, p2, [x19, #2, MUL VL]\n"
-      "st1w { z31.s }, p1, [x19, #3, MUL VL]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "st1w { z8.s }, p4, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z12.s }, p4, [x24]\n"
+      "st1w { z13.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z14.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z15.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z20.s }, p4, [x22]\n"
+      "st1w { z21.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z22.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z23.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z28.s }, p4, [x20]\n"
+      "st1w { z29.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z30.s }, p2, [x20, #2, MUL VL]\n"
+      "st1w { z31.s }, p1, [x20, #3, MUL VL]\n"
       "66:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 57b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 68f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "67:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "68:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp
index e8bad69ccd..59f33289b4 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -87,25 +87,25 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "cmp %x[M], #0x2\n"
       "bgt 23f\n"
       "beq 12f\n"
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
-      "mov x28, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "2:"  // Height 1: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 3f\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
       "zip1 z10.d, z11.d, z14.d\n"
@@ -123,154 +123,154 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "4:"  // Height 1: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "5:"  // Height 1: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 6f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "cbnz x27, 7f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "cbnz x28, 7f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
       "b 7f\n"
       "6:"  // Height 1: setup direct input
-      "mov x25, %x[input_ptr]\n"
+      "mov x26, %x[input_ptr]\n"
       "7:"  // Height 1: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 9f\n"
       "8:"  // Height 1: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "sub x26, x26, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "cmp x26, #0x10\n"
-      "add x25, x25, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
       "bgt 8b\n"
       "9:"  // Height 1: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "subs x26, x26, #0x8\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "ble 10f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "10:"  // Height 1: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 5b\n"
       "uzp1 z8.d, z8.d, z12.d\n"
-      "st1w { z8.s }, p4, [x28]\n"
       "uzp1 z9.d, z9.d, z13.d\n"
+      "st1w { z8.s }, p4, [x9]\n"
       "uzp1 z10.d, z10.d, z14.d\n"
-      "st1w { z9.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z11.d, z11.d, z15.d\n"
-      "st1w { z10.s }, p2, [x28, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z9.s }, p3, [x9, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "11:"  // Height 1: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 2b\n"
       "b 68f\n"
       "12:"  // Height 2
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "13:"  // Height 2: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 14f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
       "zip2 z13.d, z10.d, z13.d\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
       "zip1 z10.d, z11.d, z14.d\n"
       "zip2 z14.d, z11.d, z14.d\n"
       "zip1 z11.d, z16.d, z15.d\n"
@@ -286,179 +286,179 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "15:"  // Height 2: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "16:"  // Height 2: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 17f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "cbnz x27, 18f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "cbnz x28, 18f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
       "b 18f\n"
       "17:"  // Height 2: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
       "18:"  // Height 2: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 20f\n"
       "19:"  // Height 2: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "trn2 z1.d, z1.d, z2.d\n"
-      "add x24, x24, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
+      "cmp x27, #0x10\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      "add x26, x26, #0x10\n"
+      "add x25, x25, #0x10\n"
       "bgt 19b\n"
       "20:"  // Height 2: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "trn2 z1.d, z1.d, z2.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
+      "trn2 z1.d, z1.d, z2.d\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "ble 21f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      "addvl x10, x10, #8\n"
       "21:"  // Height 2: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 16b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
-      "addvl x28, x28, #4\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
+      "st1w { z8.s }, p4, [x24]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "22:"  // Height 2: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 13b\n"
       "b 68f\n"
       "23:"  // Height 3
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "24:"  // Height 3: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 25f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
       "zip2 z15.d, z16.d, z15.d\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
       "zip2 z20.d, z17.d, z20.d\n"
       "zip1 z17.d, z18.d, z21.d\n"
@@ -486,239 +486,239 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "26:"  // Height 3: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "27:"  // Height 3: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 28f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "cbnz x27, 29f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "cbnz x28, 29f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
       "b 29f\n"
       "28:"  // Height 3: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
       "29:"  // Height 3: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 31f\n"
       "30:"  // Height 3: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "add x25, x25, #0x10\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       "bgt 30b\n"
       "31:"  // Height 3: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       "ble 32f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       "32:"  // Height 3: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 27b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z16.d, z16.d, z20.d\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp1 z17.d, z17.d, z21.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
       "uzp1 z18.d, z18.d, z22.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp1 z19.d, z19.d, z23.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x22]\n"
-      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x23]\n"
+      "st1w { z17.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x23, #3, MUL VL]\n"
       "33:"  // Height 3: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 24b\n"
       "b 68f\n"
       "34:"  // Height 4
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "35:"  // Height 4: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 36f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
       "zip2 z21.d, z18.d, z21.d\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
       "zip2 z22.d, z19.d, z22.d\n"
       "zip1 z19.d, z24.d, z23.d\n"
@@ -742,263 +742,263 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "37:"  // Height 4: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "38:"  // Height 4: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 39f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "cbnz x27, 40f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "cbnz x28, 40f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
       "b 40f\n"
       "39:"  // Height 4: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
       "40:"  // Height 4: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 42f\n"
       "41:"  // Height 4: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "cmp x26, #0x10\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x24, x24, #0x10\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "add x23, x23, #0x10\n"
-      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
-      "add x22, x22, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
+      "cmp x27, #0x10\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
+      "add x23, x23, #0x10\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       "bgt 41b\n"
       "42:"  // Height 4: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
+      "trn2 z3.d, z3.d, z4.d\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       "ble 43f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       "43:"  // Height 4: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 38b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
-      "st1w { z16.s }, p4, [x21]\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
       "44:"  // Height 4: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 35b\n"
       "b 68f\n"
       "45:"  // Height 5
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "mov x9, %x[output_ptr]\n"
       "46:"  // Height 5: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 47f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z25.s }, p4/Z, [x20]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x21]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
+      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
       "zip2 z23.d, z24.d, z23.d\n"
+      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
@@ -1034,115 +1034,115 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "48:"  // Height 5: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "49:"  // Height 5: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 50f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "cbnz x27, 51f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "cbnz x28, 51f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
       "b 51f\n"
       "50:"  // Height 5: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
       "51:"  // Height 5: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 53f\n"
       "52:"  // Height 5: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "sub x26, x26, #0x10\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "cmp x26, #0x10\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x25, x25, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "add x24, x24, #0x10\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x23, x23, #0x10\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
-      "add x22, x22, #0x10\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
-      "add x21, x21, #0x10\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
       ".inst 0x45c79898  // ummla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x45c6989c  // ummla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
       ".inst 0x45c79899  // ummla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x45c6989d  // ummla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
       ".inst 0x45c7989a  // ummla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c6989e  // ummla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c7989b  // ummla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       ".inst 0x45c6989f  // ummla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
       ".inst 0x45c798b8  // ummla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
       ".inst 0x45c698bc  // ummla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
       ".inst 0x45c798b9  // ummla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
       ".inst 0x45c698bd  // ummla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
       ".inst 0x45c798ba  // ummla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
       ".inst 0x45c698be  // ummla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c798bb  // ummla z27.s, z5.b, z7.b\n"
@@ -1151,80 +1151,80 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       ".inst 0x45c698bf  // ummla z31.s, z5.b, z6.b\n"
       "bgt 52b\n"
       "53:"  // Height 5: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "subs x26, x26, #0x8\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
       "trn1 z2.d, z3.d, z4.d\n"
       "trn2 z3.d, z3.d, z4.d\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
       ".inst 0x45c79898  // ummla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
       ".inst 0x45c6989c  // ummla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
       ".inst 0x45c79899  // ummla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
       ".inst 0x45c6989d  // ummla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
       ".inst 0x45c7989a  // ummla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c6989e  // ummla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c7989b  // ummla z27.s, z4.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       ".inst 0x45c6989f  // ummla z31.s, z4.b, z6.b\n"
       "ble 54f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
       ".inst 0x45c798b8  // ummla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
       ".inst 0x45c698bc  // ummla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
       ".inst 0x45c798b9  // ummla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
       ".inst 0x45c698bd  // ummla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
       ".inst 0x45c798ba  // ummla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
       ".inst 0x45c698be  // ummla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c798bb  // ummla z27.s, z5.b, z7.b\n"
@@ -1232,127 +1232,127 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       ".inst 0x45c698bf  // ummla z31.s, z5.b, z6.b\n"
       "54:"  // Height 5: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 49b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "addvl x28, x28, #4\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
       "uzp1 z24.d, z24.d, z28.d\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp1 z25.d, z25.d, z29.d\n"
-      "st1w { z16.s }, p4, [x21]\n"
       "uzp1 z26.d, z26.d, z30.d\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "uzp1 z27.d, z27.d, z31.d\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x20]\n"
-      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x21]\n"
+      "st1w { z25.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x21, #3, MUL VL]\n"
       "55:"  // Height 5: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 46b\n"
       "b 68f\n"
       "56:"  // Height 6
-      "ldr x10, [%x[args_ptr], %[offsetof_N]]\n"
-      "mov x28, %x[output_ptr]\n"
-      "ldr x9, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n"
       "mov x20, #0x18\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "madd %x[output_ptr], x19, x20, %x[output_ptr]\n"
+      "ldr x11, [%x[args_ptr], %[offsetof_N]]\n"
+      "mov x9, %x[output_ptr]\n"
+      "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+      "madd %x[output_ptr], x21, x20, %x[output_ptr]\n"
       "57:"  // Height 6: Column loop
-      "mov x19, #0x0\n"
-      "whilelt p4.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p3.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p2.s, x19, x10\n"
-      "incw x19\n"
-      "whilelt p1.s, x19, x10\n"
+      "mov x20, #0x0\n"
+      "whilelt p4.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p3.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p2.s, x20, x11\n"
+      "incw x20\n"
+      "whilelt p1.s, x20, x11\n"
       "tbz %x[flags], #0, 58f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
-      "ld1w { z9.s }, p4/Z, [x28]\n"
-      "add x23, x28, x19, LSL #2\n"
-      "ld1w { z10.s }, p3/Z, [x28, #1, MUL VL]\n"
-      "ld1w { z11.s }, p2/Z, [x28, #2, MUL VL]\n"
-      "add x22, x23, x19, LSL #2\n"
-      "ld1w { z16.s }, p1/Z, [x28, #3, MUL VL]\n"
-      "add x21, x22, x19, LSL #2\n"
-      "ld1w { z12.s }, p4/Z, [x23]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
+      "ld1w { z9.s }, p4/Z, [x9]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
+      "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n"
+      "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n"
+      "add x20, x21, x20, LSL #2\n"
+      "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n"
+      "ld1w { z12.s }, p4/Z, [x24]\n"
       "zip1 z8.d, z9.d, z12.d\n"
-      "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n"
-      "add x20, x21, x19, LSL #2\n"
+      "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n"
+      "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n"
       "zip2 z12.d, z9.d, z12.d\n"
-      "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n"
-      "add x19, x20, x19, LSL #2\n"
       "zip1 z9.d, z10.d, z13.d\n"
-      "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n"
+      "ld1w { z17.s }, p4/Z, [x23]\n"
       "zip2 z13.d, z10.d, z13.d\n"
-      "ld1w { z17.s }, p4/Z, [x22]\n"
       "zip1 z10.d, z11.d, z14.d\n"
-      "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n"
+      "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n"
       "zip2 z14.d, z11.d, z14.d\n"
-      "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip1 z11.d, z16.d, z15.d\n"
-      "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n"
+      "ld1w { z20.s }, p4/Z, [x22]\n"
       "zip2 z15.d, z16.d, z15.d\n"
-      "ld1w { z20.s }, p4/Z, [x21]\n"
-      "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n"
       "zip1 z16.d, z17.d, z20.d\n"
-      "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n"
+      "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n"
       "zip2 z20.d, z17.d, z20.d\n"
-      "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n"
       "zip1 z17.d, z18.d, z21.d\n"
-      "ld1w { z25.s }, p4/Z, [x20]\n"
+      "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n"
+      "ld1w { z25.s }, p4/Z, [x21]\n"
       "zip2 z21.d, z18.d, z21.d\n"
-      "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n"
       "zip1 z18.d, z19.d, z22.d\n"
-      "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n"
+      "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n"
       "zip2 z22.d, z19.d, z22.d\n"
-      "ld1w { z6.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip1 z19.d, z24.d, z23.d\n"
-      "ld1w { z28.s }, p4/Z, [x19]\n"
+      "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n"
+      "ld1w { z28.s }, p4/Z, [x20]\n"
       "zip2 z23.d, z24.d, z23.d\n"
-      "ld1w { z29.s }, p3/Z, [x19, #1, MUL VL]\n"
-      "ld1w { z30.s }, p2/Z, [x19, #2, MUL VL]\n"
       "zip1 z24.d, z25.d, z28.d\n"
-      "ld1w { z31.s }, p1/Z, [x19, #3, MUL VL]\n"
+      "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n"
       "zip2 z28.d, z25.d, z28.d\n"
       "zip1 z25.d, z26.d, z29.d\n"
+      "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n"
       "zip2 z29.d, z26.d, z29.d\n"
       "zip1 z26.d, z27.d, z30.d\n"
       "zip2 z30.d, z27.d, z30.d\n"
@@ -1385,120 +1385,120 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       "mov z30.s, #0x0\n"
       "mov z31.s, #0x0\n"
       "59:"  // Height 6: setup done
-      "mov x27, #0x0\n"
+      "mov x28, #0x0\n"
       "60:"  // Height 6: String loop
       "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
-      "ldr w26, [x20, x27, LSL #0x2]\n"
+      "ldr w27, [x20, x28, LSL #0x2]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 61f\n"
-      "ldr x20, [%x[input_ptr], x27, LSL #0x3]\n"
-      "add x20, x20, x19, LSL #3\n"
-      "ldr x25, [x20, #0x0]\n"
-      "ldr x24, [x20, #0x8]\n"
-      "ldr x23, [x20, #0x10]\n"
-      "ldr x22, [x20, #0x18]\n"
-      "ldr x21, [x20, #0x20]\n"
-      "ldr x20, [x20, #0x28]\n"
-      "cbnz x27, 62f\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
-      "add x25, x25, x19\n"
-      "add x24, x24, x19\n"
-      "add x23, x23, x19\n"
-      "add x22, x22, x19\n"
-      "add x21, x21, x19\n"
-      "add x20, x20, x19\n"
+      "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n"
+      "add x21, x21, x20, LSL #3\n"
+      "ldr x26, [x21, #0x0]\n"
+      "ldr x25, [x21, #0x8]\n"
+      "ldr x24, [x21, #0x10]\n"
+      "ldr x23, [x21, #0x18]\n"
+      "ldr x22, [x21, #0x20]\n"
+      "ldr x21, [x21, #0x28]\n"
+      "cbnz x28, 62f\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+      "add x26, x26, x20\n"
+      "add x25, x25, x20\n"
+      "add x24, x24, x20\n"
+      "add x23, x23, x20\n"
+      "add x22, x22, x20\n"
+      "add x21, x21, x20\n"
       "b 62f\n"
       "61:"  // Height 6: setup direct input
-      "mov x25, %x[input_ptr]\n"
-      "add x24, x25, x19\n"
-      "add x23, x24, x19\n"
-      "add x22, x23, x19\n"
-      "add x21, x22, x19\n"
-      "add x20, x21, x19\n"
+      "mov x26, %x[input_ptr]\n"
+      "add x25, x26, x20\n"
+      "add x24, x25, x20\n"
+      "add x23, x24, x20\n"
+      "add x22, x23, x20\n"
+      "add x21, x22, x20\n"
       "62:"  // Height 6: input setup done
-      "cmp x26, #0x10\n"
+      "cmp x27, #0x10\n"
       "ble 64f\n"
       "63:"  // Height 6: Multiply loop: Main loop head
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "sub x26, x26, #0x10\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
-      "cmp x26, #0x10\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "add x25, x25, #0x10\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      "add x24, x24, #0x10\n"
       "trn1 z2.d, z3.d, z4.d\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
-      "add x23, x23, #0x10\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
-      "add x22, x22, #0x10\n"
-      "add x21, x21, #0x10\n"
-      ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
-      "add x20, x20, #0x10\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
+      ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
       ".inst 0x45c79898  // ummla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "sub x27, x27, #0x10\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
+      "cmp x27, #0x10\n"
+      "add x26, x26, #0x10\n"
       ".inst 0x45c6989c  // ummla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
       ".inst 0x45c79899  // ummla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
+      "add x24, x24, #0x10\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
+      "add x23, x23, #0x10\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x45c6989d  // ummla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
       ".inst 0x45c7989a  // ummla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c6989e  // ummla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #16\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #16\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c7989b  // ummla z27.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       ".inst 0x45c6989f  // ummla z31.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
       ".inst 0x45c798b8  // ummla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
       ".inst 0x45c698bc  // ummla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
       ".inst 0x45c798b9  // ummla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
       ".inst 0x45c698bd  // ummla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
       ".inst 0x45c798ba  // ummla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #-2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
       ".inst 0x45c698be  // ummla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #-1, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c798bb  // ummla z27.s, z5.b, z7.b\n"
@@ -1507,81 +1507,81 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       ".inst 0x45c698bf  // ummla z31.s, z5.b, z6.b\n"
       "bgt 63b\n"
       "64:"  // Height 6: Multiply loop: Single iteration only
-      "ld1b { z7.b }, p5/Z, [x9]\n"
-      "whilelt p0.b, XZR, x26\n"
-      "subs x26, x26, #0x8\n"
-      "ld1rqb { z1.b }, p0/Z, [x25]\n"
-      "ld1rqb { z2.b }, p0/Z, [x24]\n"
+      "whilelt p0.b, XZR, x27\n"
+      "ld1rqb { z1.b }, p0/Z, [x26]\n"
+      "ld1rqb { z2.b }, p0/Z, [x25]\n"
       "trn1 z0.d, z1.d, z2.d\n"
-      "ld1rqb { z3.b }, p0/Z, [x23]\n"
+      "ld1rqb { z3.b }, p0/Z, [x24]\n"
+      "ld1rqb { z4.b }, p0/Z, [x23]\n"
       "trn2 z1.d, z1.d, z2.d\n"
-      "ld1rqb { z4.b }, p0/Z, [x22]\n"
-      "ld1rqb { z5.b }, p0/Z, [x21]\n"
-      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
-      "ld1rqb { z6.b }, p0/Z, [x20]\n"
       "trn1 z2.d, z3.d, z4.d\n"
+      "ld1rqb { z5.b }, p0/Z, [x22]\n"
+      "ld1rqb { z6.b }, p0/Z, [x21]\n"
       "trn2 z3.d, z3.d, z4.d\n"
       "trn1 z4.d, z5.d, z6.d\n"
       "trn2 z5.d, z5.d, z6.d\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
+      ".inst 0x45c79808  // ummla z8.s, z0.b, z7.b\n"
       ".inst 0x45c79850  // ummla z16.s, z2.b, z7.b\n"
       ".inst 0x45c79898  // ummla z24.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
+      "subs x27, x27, #0x8\n"
       ".inst 0x45c6980c  // ummla z12.s, z0.b, z6.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
       ".inst 0x45c6989c  // ummla z28.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79809  // ummla z9.s, z0.b, z7.b\n"
       ".inst 0x45c79851  // ummla z17.s, z2.b, z7.b\n"
       ".inst 0x45c79899  // ummla z25.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c6980d  // ummla z13.s, z0.b, z6.b\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
       ".inst 0x45c6989d  // ummla z29.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7980a  // ummla z10.s, z0.b, z7.b\n"
       ".inst 0x45c79852  // ummla z18.s, z2.b, z7.b\n"
       ".inst 0x45c7989a  // ummla z26.s, z4.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c6980e  // ummla z14.s, z0.b, z6.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c6989e  // ummla z30.s, z4.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c79853  // ummla z19.s, z2.b, z7.b\n"
       ".inst 0x45c7989b  // ummla z27.s, z4.b, z7.b\n"
       ".inst 0x45c6980f  // ummla z15.s, z0.b, z6.b\n"
       ".inst 0x45c69857  // ummla z23.s, z2.b, z6.b\n"
       ".inst 0x45c6989f  // ummla z31.s, z4.b, z6.b\n"
       "ble 65f\n"
-      "ld1b { z7.b }, p5/Z, [x9]\n"
+      "ld1b { z7.b }, p5/Z, [x10]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n"
       ".inst 0x45c79828  // ummla z8.s, z1.b, z7.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #1, MUL VL]\n"
       ".inst 0x45c79870  // ummla z16.s, z3.b, z7.b\n"
       ".inst 0x45c798b8  // ummla z24.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #2, MUL VL]\n"
       ".inst 0x45c6982c  // ummla z12.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n"
       ".inst 0x45c69874  // ummla z20.s, z3.b, z6.b\n"
       ".inst 0x45c698bc  // ummla z28.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #3, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n"
       ".inst 0x45c79829  // ummla z9.s, z1.b, z7.b\n"
       ".inst 0x45c79871  // ummla z17.s, z3.b, z7.b\n"
       ".inst 0x45c798b9  // ummla z25.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #4, MUL VL]\n"
       ".inst 0x45c6982d  // ummla z13.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n"
       ".inst 0x45c69875  // ummla z21.s, z3.b, z6.b\n"
       ".inst 0x45c698bd  // ummla z29.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #5, MUL VL]\n"
+      "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n"
       ".inst 0x45c7982a  // ummla z10.s, z1.b, z7.b\n"
       ".inst 0x45c79872  // ummla z18.s, z3.b, z7.b\n"
       ".inst 0x45c798ba  // ummla z26.s, z5.b, z7.b\n"
-      "ld1b { z7.b }, p5/Z, [x9, #6, MUL VL]\n"
       ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n"
       ".inst 0x45c69876  // ummla z22.s, z3.b, z6.b\n"
       ".inst 0x45c698be  // ummla z30.s, z5.b, z6.b\n"
-      "ld1b { z6.b }, p5/Z, [x9, #7, MUL VL]\n"
-      "addvl x9, x9, #8\n"
+      "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n"
+      "addvl x10, x10, #8\n"
       ".inst 0x45c7982b  // ummla z11.s, z1.b, z7.b\n"
       ".inst 0x45c79873  // ummla z19.s, z3.b, z7.b\n"
       ".inst 0x45c798bb  // ummla z27.s, z5.b, z7.b\n"
@@ -1589,85 +1589,85 @@ void sve_hybrid_u8u32_mmla_6x4VL (
       ".inst 0x45c69877  // ummla z23.s, z3.b, z6.b\n"
       ".inst 0x45c698bf  // ummla z31.s, z5.b, z6.b\n"
       "65:"  // Height 6: Multiply loop: multiply skip
-      "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
-      "add x27, x27, #0x1\n"
-      "cmp x27, x19\n"
+      "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+      "add x28, x28, #0x1\n"
+      "cmp x28, x20\n"
       "bne 60b\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x24, x9, x20, LSL #2\n"
+      "add x23, x24, x20, LSL #2\n"
       "uzp1 z7.d, z8.d, z12.d\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+      "add x22, x23, x20, LSL #2\n"
+      "add x21, x22, x20, LSL #2\n"
       "uzp2 z8.d, z8.d, z12.d\n"
-      "st1w { z7.s }, p4, [x28]\n"
       "uzp1 z12.d, z9.d, z13.d\n"
-      "add x23, x28, x19, LSL #2\n"
+      "add x20, x21, x20, LSL #2\n"
       "uzp2 z9.d, z9.d, z13.d\n"
-      "st1w { z12.s }, p3, [x28, #1, MUL VL]\n"
       "uzp1 z13.d, z10.d, z14.d\n"
-      "add x22, x23, x19, LSL #2\n"
+      "st1w { z7.s }, p4, [x9]\n"
       "uzp2 z10.d, z10.d, z14.d\n"
-      "st1w { z13.s }, p2, [x28, #2, MUL VL]\n"
       "uzp1 z14.d, z11.d, z15.d\n"
-      "add x21, x22, x19, LSL #2\n"
+      "st1w { z12.s }, p3, [x9, #1, MUL VL]\n"
       "uzp2 z11.d, z11.d, z15.d\n"
-      "st1w { z14.s }, p1, [x28, #3, MUL VL]\n"
       "uzp1 z15.d, z16.d, z20.d\n"
-      "add x20, x21, x19, LSL #2\n"
+      "st1w { z13.s }, p2, [x9, #2, MUL VL]\n"
       "uzp2 z16.d, z16.d, z20.d\n"
-      "st1w { z8.s }, p4, [x23]\n"
-      "add x19, x20, x19, LSL #2\n"
       "uzp1 z20.d, z17.d, z21.d\n"
-      "st1w { z9.s }, p3, [x23, #1, MUL VL]\n"
-      "addvl x28, x28, #4\n"
+      "st1w { z14.s }, p1, [x9, #3, MUL VL]\n"
+      "addvl x9, x9, #4\n"
       "uzp2 z17.d, z17.d, z21.d\n"
-      "st1w { z10.s }, p2, [x23, #2, MUL VL]\n"
       "uzp1 z21.d, z18.d, z22.d\n"
-      "st1w { z11.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z8.s }, p4, [x24]\n"
       "uzp2 z18.d, z18.d, z22.d\n"
-      "st1w { z15.s }, p4, [x22]\n"
       "uzp1 z22.d, z19.d, z23.d\n"
-      "st1w { z20.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z9.s }, p3, [x24, #1, MUL VL]\n"
       "uzp2 z19.d, z19.d, z23.d\n"
-      "st1w { z21.s }, p2, [x22, #2, MUL VL]\n"
       "uzp1 z23.d, z24.d, z28.d\n"
-      "st1w { z22.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z10.s }, p2, [x24, #2, MUL VL]\n"
       "uzp2 z24.d, z24.d, z28.d\n"
-      "st1w { z16.s }, p4, [x21]\n"
       "uzp1 z28.d, z25.d, z29.d\n"
-      "st1w { z17.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z11.s }, p1, [x24, #3, MUL VL]\n"
       "uzp2 z25.d, z25.d, z29.d\n"
-      "st1w { z18.s }, p2, [x21, #2, MUL VL]\n"
       "uzp1 z29.d, z26.d, z30.d\n"
-      "st1w { z19.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z15.s }, p4, [x23]\n"
       "uzp2 z26.d, z26.d, z30.d\n"
-      "st1w { z23.s }, p4, [x20]\n"
       "uzp1 z30.d, z27.d, z31.d\n"
-      "st1w { z28.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z20.s }, p3, [x23, #1, MUL VL]\n"
       "uzp2 z27.d, z27.d, z31.d\n"
-      "st1w { z29.s }, p2, [x20, #2, MUL VL]\n"
-      "st1w { z30.s }, p1, [x20, #3, MUL VL]\n"
-      "st1w { z24.s }, p4, [x19]\n"
-      "st1w { z25.s }, p3, [x19, #1, MUL VL]\n"
-      "st1w { z26.s }, p2, [x19, #2, MUL VL]\n"
-      "st1w { z27.s }, p1, [x19, #3, MUL VL]\n"
+      "st1w { z21.s }, p2, [x23, #2, MUL VL]\n"
+      "st1w { z22.s }, p1, [x23, #3, MUL VL]\n"
+      "st1w { z16.s }, p4, [x22]\n"
+      "st1w { z17.s }, p3, [x22, #1, MUL VL]\n"
+      "st1w { z18.s }, p2, [x22, #2, MUL VL]\n"
+      "st1w { z19.s }, p1, [x22, #3, MUL VL]\n"
+      "st1w { z23.s }, p4, [x21]\n"
+      "st1w { z28.s }, p3, [x21, #1, MUL VL]\n"
+      "st1w { z29.s }, p2, [x21, #2, MUL VL]\n"
+      "st1w { z30.s }, p1, [x21, #3, MUL VL]\n"
+      "st1w { z24.s }, p4, [x20]\n"
+      "st1w { z25.s }, p3, [x20, #1, MUL VL]\n"
+      "st1w { z26.s }, p2, [x20, #2, MUL VL]\n"
+      "st1w { z27.s }, p1, [x20, #3, MUL VL]\n"
       "66:"  // Height 6: Writeback done
-      "decw x10, ALL, MUL #4\n"
-      "cmp x10, XZR\n"
+      "decw x11, ALL, MUL #4\n"
+      "cmp x11, XZR\n"
       "bgt 57b\n"
       "subs %x[M], %x[M], #0x6\n"
       "beq 68f\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "tbz %x[flags], #3, 67f\n"
-      "add x20, x20, #0x6\n"
-      "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+      "add x21, x21, #0x6\n"
+      "str x21, [%x[args_ptr], %[offsetof_input_offset]]\n"
       "b 1b\n"
       "67:"  // Update direct input
-      "mov x19, #0x6\n"
-      "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+      "mov x20, #0x6\n"
+      "madd %x[input_ptr], x20, x21, %x[input_ptr]\n"
       "b 1b\n"
       "68:"  // Exit
 
       : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr)
       : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
-      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
index e604dcc4bc..94452929c6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,42 +33,44 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const bfloat16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/2) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "mov %x[Apanel], x21\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
+      "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "mov z12.b, #0x0\n"
       "mov z13.b, #0x0\n"
-      "mov %x[Apanel], x21\n"
+      "ld1h { z4.h }, p0/Z, [x22]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
-      "cmp x19, #0x2\n"
+      "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.b, #0x0\n"
       "mov z17.b, #0x0\n"
+      "ld1h { z6.h }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
-      "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "mov z20.b, #0x0\n"
       "mov z21.b, #0x0\n"
-      "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "mov z24.b, #0x0\n"
@@ -83,31 +85,29 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       "3:"  // main loop head
       ".inst 0x64604088  // bfdot z8.s, z4.h, z0.h[0]\n"
       ".inst 0x6468408b  // bfdot z11.s, z4.h, z0.h[1]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
       ".inst 0x6470408e  // bfdot z14.s, z4.h, z0.h[2]\n"
       ".inst 0x64784091  // bfdot z17.s, z4.h, z0.h[3]\n"
-      "ld1h { z6.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n"
       ".inst 0x64614094  // bfdot z20.s, z4.h, z1.h[0]\n"
       ".inst 0x64694097  // bfdot z23.s, z4.h, z1.h[1]\n"
-      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6471409a  // bfdot z26.s, z4.h, z1.h[2]\n"
       ".inst 0x6479409d  // bfdot z29.s, z4.h, z1.h[3]\n"
-      "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x646040a9  // bfdot z9.s, z5.h, z0.h[0]\n"
       ".inst 0x646840ac  // bfdot z12.s, z5.h, z0.h[1]\n"
-      "ld1h { z4.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "cmp x20, #0x2\n"
       ".inst 0x647040af  // bfdot z15.s, z5.h, z0.h[2]\n"
       ".inst 0x647840b2  // bfdot z18.s, z5.h, z0.h[3]\n"
-      "sub x19, x19, #0x2\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x646140b5  // bfdot z21.s, z5.h, z1.h[0]\n"
       ".inst 0x646940b8  // bfdot z24.s, z5.h, z1.h[1]\n"
-      "cmp x19, #0x2\n"
       ".inst 0x647140bb  // bfdot z27.s, z5.h, z1.h[2]\n"
       ".inst 0x647940be  // bfdot z30.s, z5.h, z1.h[3]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #4, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #4, MUL VL]\n"
       ".inst 0x646040ca  // bfdot z10.s, z6.h, z0.h[0]\n"
       ".inst 0x646840cd  // bfdot z13.s, z6.h, z0.h[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x647040d0  // bfdot z16.s, z6.h, z0.h[2]\n"
       ".inst 0x647840d3  // bfdot z19.s, z6.h, z0.h[3]\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
@@ -115,26 +115,27 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       ".inst 0x646940d9  // bfdot z25.s, z6.h, z1.h[1]\n"
       ".inst 0x647140dc  // bfdot z28.s, z6.h, z1.h[2]\n"
       ".inst 0x647940df  // bfdot z31.s, z6.h, z1.h[3]\n"
-      "ld1h { z6.h }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #5, MUL VL]\n"
+      "addvl x22, x22, #6\n"
       ".inst 0x64624088  // bfdot z8.s, z4.h, z2.h[0]\n"
       ".inst 0x646a408b  // bfdot z11.s, z4.h, z2.h[1]\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       ".inst 0x6472408e  // bfdot z14.s, z4.h, z2.h[2]\n"
       ".inst 0x647a4091  // bfdot z17.s, z4.h, z2.h[3]\n"
-      "addvl x20, x20, #6\n"
       ".inst 0x64634094  // bfdot z20.s, z4.h, z3.h[0]\n"
       ".inst 0x646b4097  // bfdot z23.s, z4.h, z3.h[1]\n"
       ".inst 0x6473409a  // bfdot z26.s, z4.h, z3.h[2]\n"
       ".inst 0x647b409d  // bfdot z29.s, z4.h, z3.h[3]\n"
+      "ld1h { z4.h }, p0/Z, [x22]\n"
       ".inst 0x646240a9  // bfdot z9.s, z5.h, z2.h[0]\n"
       ".inst 0x646a40ac  // bfdot z12.s, z5.h, z2.h[1]\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
       ".inst 0x647240af  // bfdot z15.s, z5.h, z2.h[2]\n"
       ".inst 0x647a40b2  // bfdot z18.s, z5.h, z2.h[3]\n"
       ".inst 0x646340b5  // bfdot z21.s, z5.h, z3.h[0]\n"
       ".inst 0x646b40b8  // bfdot z24.s, z5.h, z3.h[1]\n"
       ".inst 0x647340bb  // bfdot z27.s, z5.h, z3.h[2]\n"
       ".inst 0x647b40be  // bfdot z30.s, z5.h, z3.h[3]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x646240ca  // bfdot z10.s, z6.h, z2.h[0]\n"
       ".inst 0x646a40cd  // bfdot z13.s, z6.h, z2.h[1]\n"
       ".inst 0x647240d0  // bfdot z16.s, z6.h, z2.h[2]\n"
@@ -143,20 +144,19 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       ".inst 0x646b40d9  // bfdot z25.s, z6.h, z3.h[1]\n"
       ".inst 0x647340dc  // bfdot z28.s, z6.h, z3.h[2]\n"
       ".inst 0x647b40df  // bfdot z31.s, z6.h, z3.h[3]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #2, MUL VL]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       ".inst 0x64604088  // bfdot z8.s, z4.h, z0.h[0]\n"
       ".inst 0x6468408b  // bfdot z11.s, z4.h, z0.h[1]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       ".inst 0x6470408e  // bfdot z14.s, z4.h, z0.h[2]\n"
       ".inst 0x64784091  // bfdot z17.s, z4.h, z0.h[3]\n"
-      "ld1h { z6.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "addvl x22, x22, #3\n"
       ".inst 0x64614094  // bfdot z20.s, z4.h, z1.h[0]\n"
       ".inst 0x64694097  // bfdot z23.s, z4.h, z1.h[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
       ".inst 0x6471409a  // bfdot z26.s, z4.h, z1.h[2]\n"
       ".inst 0x6479409d  // bfdot z29.s, z4.h, z1.h[3]\n"
-      "addvl x20, x20, #3\n"
       ".inst 0x646040a9  // bfdot z9.s, z5.h, z0.h[0]\n"
       ".inst 0x646840ac  // bfdot z12.s, z5.h, z0.h[1]\n"
       ".inst 0x647040af  // bfdot z15.s, z5.h, z0.h[2]\n"
@@ -173,19 +173,19 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       ".inst 0x646940d9  // bfdot z25.s, z6.h, z1.h[1]\n"
       ".inst 0x647140dc  // bfdot z28.s, z6.h, z1.h[2]\n"
       ".inst 0x647940df  // bfdot z31.s, z6.h, z1.h[3]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ld1h { z7.h }, p0/Z, [x20]\n"
-      "ld1h { z4.h }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #2, MUL VL]\n"
-      "addvl x20, x20, #3\n"
+      "ld1h { z7.h }, p0/Z, [x22]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x646040e8  // bfdot z8.s, z7.h, z0.h[0]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #2, MUL VL]\n"
       ".inst 0x646840eb  // bfdot z11.s, z7.h, z0.h[1]\n"
       ".inst 0x647040ee  // bfdot z14.s, z7.h, z0.h[2]\n"
       ".inst 0x647840f1  // bfdot z17.s, z7.h, z0.h[3]\n"
       ".inst 0x646140f4  // bfdot z20.s, z7.h, z1.h[0]\n"
+      "addvl x22, x22, #3\n"
       ".inst 0x646940f7  // bfdot z23.s, z7.h, z1.h[1]\n"
       ".inst 0x647140fa  // bfdot z26.s, z7.h, z1.h[2]\n"
       ".inst 0x647940fd  // bfdot z29.s, z7.h, z1.h[3]\n"
@@ -207,7 +207,7 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       ".inst 0x647940bf  // bfdot z31.s, z5.h, z1.h[3]\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -238,7 +238,7 @@ void sve_interleaved_bf16fp32_dot_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
index de4f0ad313..fe5382db05 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,28 +33,28 @@ void sve_interleaved_bf16fp32_mmla_8x3VL(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const bfloat16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ld1h { z4.h }, p0/Z, [x20]\n"
+      "ld1h { z4.h }, p0/Z, [x22]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
@@ -63,13 +63,13 @@ void sve_interleaved_bf16fp32_mmla_8x3VL(
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
-      "ld1h { z5.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.b, #0x0\n"
       "mov z17.b, #0x0\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
-      "addvl x20, x20, #2\n"
+      "addvl x22, x22, #2\n"
       "mov z20.b, #0x0\n"
       "mov z21.b, #0x0\n"
       "add %x[Apanel], %x[Apanel], #0x30\n"
@@ -87,143 +87,143 @@ void sve_interleaved_bf16fp32_mmla_8x3VL(
       "3:"  // main loop head
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
-      ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e40b  // bfmmla z11.s, z0.h, z5.h\n"
+      ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e431  // bfmmla z17.s, z1.h, z5.h\n"
-      "ld1h { z6.h }, p0/Z, [x20]\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e457  // bfmmla z23.s, z2.h, z5.h\n"
-      "ld1h { z7.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x6464e47a  // bfmmla z26.s, z3.h, z4.h\n"
       ".inst 0x6465e47d  // bfmmla z29.s, z3.h, z5.h\n"
-      "ld1h { z4.h }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
+      ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      ".inst 0x6467e432  // bfmmla z18.s, z1.h, z7.h\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
+      "cmp x20, #0x2\n"
       ".inst 0x6466e47b  // bfmmla z27.s, z3.h, z6.h\n"
-      "ld1h { z6.h }, p0/Z, [x20, #4, MUL VL]\n"
-      ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
+      ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
+      "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x6465e40d  // bfmmla z13.s, z0.h, z5.h\n"
-      ".inst 0x6467e432  // bfmmla z18.s, z1.h, z7.h\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #16]\n"
       ".inst 0x6464e430  // bfmmla z16.s, z1.h, z4.h\n"
       ".inst 0x6465e433  // bfmmla z19.s, z1.h, z5.h\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #32]\n"
-      ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
-      ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
-      "ld1h { z7.h }, p0/Z, [x20, #5, MUL VL]\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e459  // bfmmla z25.s, z2.h, z5.h\n"
-      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x6464e47c  // bfmmla z28.s, z3.h, z4.h\n"
       ".inst 0x6465e47f  // bfmmla z31.s, z3.h, z5.h\n"
+      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #48]\n"
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #64]\n"
-      "ld1h { z4.h }, p0/Z, [x20, #6, MUL VL]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #7, MUL VL]\n"
-      "addvl x20, x20, #16\n"
+      "ld1h { z4.h }, p0/Z, [x22, #6, MUL VL]\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
-      ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
-      "cmp x19, #0x2\n"
+      "ld1h { z5.h }, p0/Z, [x22, #7, MUL VL]\n"
+      "addvl x22, x22, #16\n"
       ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
+      ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6467e431  // bfmmla z17.s, z1.h, z7.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       ".inst 0x6466e47a  // bfmmla z26.s, z3.h, z6.h\n"
       ".inst 0x6467e47d  // bfmmla z29.s, z3.h, z7.h\n"
-      "ld1h { z6.h }, p0/Z, [x20, #-8, MUL VL]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #-7, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #-8, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #-7, MUL VL]\n"
       ".inst 0x6464e409  // bfmmla z9.s, z0.h, z4.h\n"
+      ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
       ".inst 0x6464e42f  // bfmmla z15.s, z1.h, z4.h\n"
+      ".inst 0x6465e432  // bfmmla z18.s, z1.h, z5.h\n"
       ".inst 0x6464e455  // bfmmla z21.s, z2.h, z4.h\n"
+      ".inst 0x6465e458  // bfmmla z24.s, z2.h, z5.h\n"
       ".inst 0x6464e47b  // bfmmla z27.s, z3.h, z4.h\n"
-      "ld1h { z4.h }, p0/Z, [x20, #-6, MUL VL]\n"
-      ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
+      ".inst 0x6465e47e  // bfmmla z30.s, z3.h, z5.h\n"
+      "ld1h { z4.h }, p0/Z, [x22, #-6, MUL VL]\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      ".inst 0x6465e432  // bfmmla z18.s, z1.h, z5.h\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #80]\n"
       ".inst 0x6466e430  // bfmmla z16.s, z1.h, z6.h\n"
       ".inst 0x6467e433  // bfmmla z19.s, z1.h, z7.h\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #96]\n"
-      ".inst 0x6465e458  // bfmmla z24.s, z2.h, z5.h\n"
-      ".inst 0x6465e47e  // bfmmla z30.s, z3.h, z5.h\n"
-      "ld1h { z5.h }, p0/Z, [x20, #-5, MUL VL]\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6467e459  // bfmmla z25.s, z2.h, z7.h\n"
-      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #112]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #-5, MUL VL]\n"
       ".inst 0x6466e47c  // bfmmla z28.s, z3.h, z6.h\n"
       ".inst 0x6467e47f  // bfmmla z31.s, z3.h, z7.h\n"
+      "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #112]\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "addvl x20, x20, #-4\n"
+      "addvl x22, x22, #-4\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x6464e408  // bfmmla z8.s, z0.h, z4.h\n"
-      ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e40b  // bfmmla z11.s, z0.h, z5.h\n"
+      ".inst 0x6464e42e  // bfmmla z14.s, z1.h, z4.h\n"
       ".inst 0x6465e431  // bfmmla z17.s, z1.h, z5.h\n"
-      "ld1h { z6.h }, p0/Z, [x20]\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
       ".inst 0x6464e454  // bfmmla z20.s, z2.h, z4.h\n"
       ".inst 0x6465e457  // bfmmla z23.s, z2.h, z5.h\n"
-      "ld1h { z7.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x6464e47a  // bfmmla z26.s, z3.h, z4.h\n"
       ".inst 0x6465e47d  // bfmmla z29.s, z3.h, z5.h\n"
-      "ld1h { z4.h }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x6466e409  // bfmmla z9.s, z0.h, z6.h\n"
+      ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
       ".inst 0x6466e42f  // bfmmla z15.s, z1.h, z6.h\n"
+      ".inst 0x6467e432  // bfmmla z18.s, z1.h, z7.h\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x6466e455  // bfmmla z21.s, z2.h, z6.h\n"
+      ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
+      "addvl x22, x22, #4\n"
       ".inst 0x6466e47b  // bfmmla z27.s, z3.h, z6.h\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x6467e40c  // bfmmla z12.s, z0.h, z7.h\n"
+      ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
       ".inst 0x6464e40a  // bfmmla z10.s, z0.h, z4.h\n"
-      "addvl x20, x20, #4\n"
       ".inst 0x6465e40d  // bfmmla z13.s, z0.h, z5.h\n"
-      ".inst 0x6467e432  // bfmmla z18.s, z1.h, z7.h\n"
       ".inst 0x6464e430  // bfmmla z16.s, z1.h, z4.h\n"
       ".inst 0x6465e433  // bfmmla z19.s, z1.h, z5.h\n"
-      ".inst 0x6467e458  // bfmmla z24.s, z2.h, z7.h\n"
-      ".inst 0x6467e47e  // bfmmla z30.s, z3.h, z7.h\n"
       ".inst 0x6464e456  // bfmmla z22.s, z2.h, z4.h\n"
       ".inst 0x6465e459  // bfmmla z25.s, z2.h, z5.h\n"
       ".inst 0x6464e47c  // bfmmla z28.s, z3.h, z4.h\n"
       ".inst 0x6465e47f  // bfmmla z31.s, z3.h, z5.h\n"
-      "cbz x19, 5f\n"
-      "ld1h { z6.h }, p0/Z, [x20]\n"
+      "cbz x20, 5f\n"
+      "ld1h { z6.h }, p0/Z, [x22]\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x6466e408  // bfmmla z8.s, z0.h, z6.h\n"
       "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #1, MUL VL]\n"
-      ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
+      "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n"
+      ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
       "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n"
       "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n"
-      ".inst 0x6467e40b  // bfmmla z11.s, z0.h, z7.h\n"
+      ".inst 0x6466e42e  // bfmmla z14.s, z1.h, z6.h\n"
       ".inst 0x6467e431  // bfmmla z17.s, z1.h, z7.h\n"
       ".inst 0x6466e454  // bfmmla z20.s, z2.h, z6.h\n"
-      "ld1h { z4.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
       ".inst 0x6467e457  // bfmmla z23.s, z2.h, z7.h\n"
       ".inst 0x6466e47a  // bfmmla z26.s, z3.h, z6.h\n"
-      "ld1h { z5.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x6467e47d  // bfmmla z29.s, z3.h, z7.h\n"
-      "ld1h { z6.h }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x6464e409  // bfmmla z9.s, z0.h, z4.h\n"
+      ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
+      "addvl x22, x22, #6\n"
       ".inst 0x6464e42f  // bfmmla z15.s, z1.h, z4.h\n"
+      ".inst 0x6465e432  // bfmmla z18.s, z1.h, z5.h\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x6464e455  // bfmmla z21.s, z2.h, z4.h\n"
+      ".inst 0x6465e458  // bfmmla z24.s, z2.h, z5.h\n"
       ".inst 0x6464e47b  // bfmmla z27.s, z3.h, z4.h\n"
-      "addvl x20, x20, #6\n"
-      ".inst 0x6465e40c  // bfmmla z12.s, z0.h, z5.h\n"
+      ".inst 0x6465e47e  // bfmmla z30.s, z3.h, z5.h\n"
       ".inst 0x6466e40a  // bfmmla z10.s, z0.h, z6.h\n"
       ".inst 0x6467e40d  // bfmmla z13.s, z0.h, z7.h\n"
-      ".inst 0x6465e432  // bfmmla z18.s, z1.h, z5.h\n"
       ".inst 0x6466e430  // bfmmla z16.s, z1.h, z6.h\n"
       ".inst 0x6467e433  // bfmmla z19.s, z1.h, z7.h\n"
-      ".inst 0x6465e458  // bfmmla z24.s, z2.h, z5.h\n"
-      ".inst 0x6465e47e  // bfmmla z30.s, z3.h, z5.h\n"
       ".inst 0x6466e456  // bfmmla z22.s, z2.h, z6.h\n"
       ".inst 0x6467e459  // bfmmla z25.s, z2.h, z7.h\n"
       ".inst 0x6466e47c  // bfmmla z28.s, z3.h, z6.h\n"
@@ -243,7 +243,7 @@ void sve_interleaved_bf16fp32_mmla_8x3VL(
       "uzp2 z14.d, z14.d, z17.d\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
       "uzp1 z17.d, z15.d, z18.d\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n"
       "uzp2 z15.d, z15.d, z18.d\n"
       "uzp1 z18.d, z16.d, z19.d\n"
@@ -285,7 +285,7 @@ void sve_interleaved_bf16fp32_mmla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp
index 602634706e..9287509889 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -32,34 +32,34 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
     __fp16 *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const __fp16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/1) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ld1h { z0.h }, p0/Z, [x20]\n"
+      "ld1h { z0.h }, p0/Z, [x22]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
-      "ld1h { z1.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z12.b, #0x0\n"
       "mov z13.b, #0x0\n"
-      "ld1h { z2.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n"
@@ -86,7 +86,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "3:"  // main loop head
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z10.h, p0/M, z2.h, z3.h\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z11.h, p0/M, z0.h, z4.h\n"
@@ -95,7 +95,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
@@ -115,11 +115,11 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "fmla z28.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z29.h, p0/M, z0.h, z6.h\n"
-      "ld1h { z0.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z0.h }, p0/Z, [x22, #3, MUL VL]\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "ld1h { z1.h }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1h { z2.h }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1h { z1.h }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1h { z2.h }, p0/Z, [x22, #5, MUL VL]\n"
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #22]\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
@@ -131,7 +131,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #26]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
-      "addvl x20, x20, #6\n"
+      "addvl x22, x22, #6\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #28]\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
@@ -151,18 +151,18 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "fmla z27.h, p0/M, z1.h, z5.h\n"
       "fmla z28.h, p0/M, z2.h, z5.h\n"
       "fmla z29.h, p0/M, z0.h, z6.h\n"
-      "ld1h { z0.h }, p0/Z, [x20]\n"
+      "ld1h { z0.h }, p0/Z, [x22]\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "ld1h { z1.h }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z2.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "fmla z9.h, p0/M, z1.h, z3.h\n"
-      "addvl x20, x20, #3\n"
+      "addvl x22, x22, #3\n"
       "fmla z10.h, p0/M, z2.h, z3.h\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z11.h, p0/M, z0.h, z4.h\n"
@@ -190,10 +190,10 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "fmla z29.h, p0/M, z0.h, z6.h\n"
       "fmla z30.h, p0/M, z1.h, z6.h\n"
       "fmla z31.h, p0/M, z2.h, z6.h\n"
-      "cbz x19, 5f\n"
-      "ld1h { z0.h }, p0/Z, [x20]\n"
-      "ld1h { z1.h }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z2.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "cbz x20, 5f\n"
+      "ld1h { z0.h }, p0/Z, [x22]\n"
+      "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.h, p0/M, z0.h, z3.h\n"
       "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n"
@@ -202,24 +202,24 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n"
       "fmla z10.h, p0/M, z2.h, z3.h\n"
       "fmla z11.h, p0/M, z0.h, z4.h\n"
-      "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z12.h, p0/M, z1.h, z4.h\n"
       "fmla z13.h, p0/M, z2.h, z4.h\n"
-      "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n"
+      "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n"
       "fmla z14.h, p0/M, z0.h, z5.h\n"
       "fmla z15.h, p0/M, z1.h, z5.h\n"
+      "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n"
       "fmla z16.h, p0/M, z2.h, z5.h\n"
       "fmla z17.h, p0/M, z0.h, z6.h\n"
       "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n"
       "fmla z18.h, p0/M, z1.h, z6.h\n"
       "fmla z19.h, p0/M, z2.h, z6.h\n"
       "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n"
-      "addvl x20, x20, #3\n"
       "fmla z20.h, p0/M, z0.h, z3.h\n"
       "fmla z21.h, p0/M, z1.h, z3.h\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
+      "addvl x22, x22, #3\n"
       "fmla z22.h, p0/M, z2.h, z3.h\n"
       "fmla z23.h, p0/M, z0.h, z4.h\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla z24.h, p0/M, z1.h, z4.h\n"
       "fmla z25.h, p0/M, z2.h, z4.h\n"
       "fmla z26.h, p0/M, z0.h, z5.h\n"
@@ -230,7 +230,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "fmla z31.h, p0/M, z2.h, z6.h\n"
       "5:"  // multiply loop done
       "st1h { z8.h }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -261,7 +261,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
index f8e4b89b95..1ac2ac075e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -32,39 +32,41 @@ void sve_interleaved_fp16_mla_8x3VL(
     __fp16 *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const __fp16 *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/1) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "mov %x[Apanel], x21\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
-      "ld1h { z2.h }, p0/Z, [x20]\n"
+      "ld1h { z2.h }, p0/Z, [x22]\n"
       "mov z12.b, #0x0\n"
       "mov z13.b, #0x0\n"
-      "mov %x[Apanel], x21\n"
+      "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
-      "cmp x19, #0x2\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z16.b, #0x0\n"
       "mov z17.b, #0x0\n"
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
-      "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "mov z20.b, #0x0\n"
       "mov z21.b, #0x0\n"
       "mov z22.b, #0x0\n"
@@ -81,34 +83,34 @@ void sve_interleaved_fp16_mla_8x3VL(
       "3:"  // main loop head
       "fmla z8.h, z2.h, z0.h[0]\n"
       "fmla z11.h, z2.h, z0.h[1]\n"
-      "ld1h { z3.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z14.h, z2.h, z0.h[2]\n"
       "fmla z17.h, z2.h, z0.h[3]\n"
-      "ld1h { z4.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n"
       "fmla z20.h, z2.h, z0.h[4]\n"
       "fmla z23.h, z2.h, z0.h[5]\n"
-      "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n"
       "fmla z26.h, z2.h, z0.h[6]\n"
       "fmla z29.h, z2.h, z0.h[7]\n"
-      "ld1h { z5.h }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n"
       "fmla z9.h, z3.h, z0.h[0]\n"
       "fmla z12.h, z3.h, z0.h[1]\n"
-      "ld1h { z6.h }, p0/Z, [x20, #4, MUL VL]\n"
+      "addvl x22, x22, #6\n"
       "fmla z15.h, z3.h, z0.h[2]\n"
       "fmla z18.h, z3.h, z0.h[3]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #5, MUL VL]\n"
+      "sub x20, x20, #0x2\n"
       "fmla z21.h, z3.h, z0.h[4]\n"
       "fmla z24.h, z3.h, z0.h[5]\n"
-      "sub x19, x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z27.h, z3.h, z0.h[6]\n"
       "fmla z30.h, z3.h, z0.h[7]\n"
-      "cmp x19, #0x2\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z10.h, z4.h, z0.h[0]\n"
       "fmla z13.h, z4.h, z0.h[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
+      "ld1h { z2.h }, p0/Z, [x22]\n"
       "fmla z16.h, z4.h, z0.h[2]\n"
       "fmla z19.h, z4.h, z0.h[3]\n"
-      "addvl x20, x20, #6\n"
+      "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n"
       "fmla z22.h, z4.h, z0.h[4]\n"
       "fmla z25.h, z4.h, z0.h[5]\n"
       "fmla z28.h, z4.h, z0.h[6]\n"
@@ -116,7 +118,7 @@ void sve_interleaved_fp16_mla_8x3VL(
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.h, z5.h, z1.h[0]\n"
       "fmla z11.h, z5.h, z1.h[1]\n"
-      "ld1h { z2.h }, p0/Z, [x20]\n"
+      "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n"
       "fmla z14.h, z5.h, z1.h[2]\n"
       "fmla z17.h, z5.h, z1.h[3]\n"
       "fmla z20.h, z5.h, z1.h[4]\n"
@@ -143,16 +145,14 @@ void sve_interleaved_fp16_mla_8x3VL(
       "4:"  // main loop skip
       "fmla z8.h, z2.h, z0.h[0]\n"
       "fmla z11.h, z2.h, z0.h[1]\n"
-      "ld1h { z3.h }, p0/Z, [x20, #1, MUL VL]\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla z14.h, z2.h, z0.h[2]\n"
       "fmla z17.h, z2.h, z0.h[3]\n"
-      "ld1h { z4.h }, p0/Z, [x20, #2, MUL VL]\n"
+      "addvl x22, x22, #3\n"
       "fmla z20.h, z2.h, z0.h[4]\n"
       "fmla z23.h, z2.h, z0.h[5]\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla z26.h, z2.h, z0.h[6]\n"
       "fmla z29.h, z2.h, z0.h[7]\n"
-      "addvl x20, x20, #3\n"
       "fmla z9.h, z3.h, z0.h[0]\n"
       "fmla z12.h, z3.h, z0.h[1]\n"
       "fmla z15.h, z3.h, z0.h[2]\n"
@@ -169,19 +169,19 @@ void sve_interleaved_fp16_mla_8x3VL(
       "fmla z25.h, z4.h, z0.h[5]\n"
       "fmla z28.h, z4.h, z0.h[6]\n"
       "fmla z31.h, z4.h, z0.h[7]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      "ld1h { z5.h }, p0/Z, [x20]\n"
-      "ld1h { z6.h }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1h { z7.h }, p0/Z, [x20, #2, MUL VL]\n"
-      "addvl x20, x20, #3\n"
+      "ld1h { z5.h }, p0/Z, [x22]\n"
       "fmla z8.h, z5.h, z0.h[0]\n"
+      "ld1h { z6.h }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1h { z7.h }, p0/Z, [x22, #2, MUL VL]\n"
       "fmla z11.h, z5.h, z0.h[1]\n"
       "fmla z14.h, z5.h, z0.h[2]\n"
       "fmla z17.h, z5.h, z0.h[3]\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       "fmla z20.h, z5.h, z0.h[4]\n"
       "fmla z23.h, z5.h, z0.h[5]\n"
+      "addvl x22, x22, #3\n"
       "fmla z26.h, z5.h, z0.h[6]\n"
       "fmla z29.h, z5.h, z0.h[7]\n"
       "fmla z9.h, z6.h, z0.h[0]\n"
@@ -202,7 +202,7 @@ void sve_interleaved_fp16_mla_8x3VL(
       "fmla z31.h, z7.h, z0.h[7]\n"
       "5:"  // multiply loop done
       "st1h { z8.h }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -233,7 +233,7 @@ void sve_interleaved_fp16_mla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp
index 6defe0e223..3141a258a8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -32,34 +32,34 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const float *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/1) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ld1w { z0.s }, p0/Z, [x20]\n"
+      "ld1w { z0.s }, p0/Z, [x22]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
-      "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z12.b, #0x0\n"
       "mov z13.b, #0x0\n"
-      "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z2.s }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
@@ -86,7 +86,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "3:"  // main loop head
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "fmla z10.s, p0/M, z2.s, z3.s\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z11.s, p0/M, z0.s, z4.s\n"
@@ -95,7 +95,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
@@ -115,11 +115,11 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "fmla z28.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n"
       "fmla z29.s, p0/M, z0.s, z6.s\n"
-      "ld1w { z0.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1w { z0.s }, p0/Z, [x22, #3, MUL VL]\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "ld1w { z1.s }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1w { z2.s }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1w { z1.s }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1w { z2.s }, p0/Z, [x22, #5, MUL VL]\n"
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
@@ -131,7 +131,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
-      "addvl x20, x20, #6\n"
+      "addvl x22, x22, #6\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
@@ -151,18 +151,18 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "fmla z27.s, p0/M, z1.s, z5.s\n"
       "fmla z28.s, p0/M, z2.s, z5.s\n"
       "fmla z29.s, p0/M, z0.s, z6.s\n"
-      "ld1w { z0.s }, p0/Z, [x20]\n"
+      "ld1w { z0.s }, p0/Z, [x22]\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z2.s }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "fmla z9.s, p0/M, z1.s, z3.s\n"
-      "addvl x20, x20, #3\n"
+      "addvl x22, x22, #3\n"
       "fmla z10.s, p0/M, z2.s, z3.s\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z11.s, p0/M, z0.s, z4.s\n"
@@ -190,10 +190,10 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "fmla z29.s, p0/M, z0.s, z6.s\n"
       "fmla z30.s, p0/M, z1.s, z6.s\n"
       "fmla z31.s, p0/M, z2.s, z6.s\n"
-      "cbz x19, 5f\n"
-      "ld1w { z0.s }, p0/Z, [x20]\n"
-      "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
+      "cbz x20, 5f\n"
+      "ld1w { z0.s }, p0/Z, [x22]\n"
+      "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1w { z2.s }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
       "fmla z8.s, p0/M, z0.s, z3.s\n"
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
@@ -202,24 +202,24 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "fmla z10.s, p0/M, z2.s, z3.s\n"
       "fmla z11.s, p0/M, z0.s, z4.s\n"
-      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z12.s, p0/M, z1.s, z4.s\n"
       "fmla z13.s, p0/M, z2.s, z4.s\n"
-      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
+      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z14.s, p0/M, z0.s, z5.s\n"
       "fmla z15.s, p0/M, z1.s, z5.s\n"
+      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "fmla z16.s, p0/M, z2.s, z5.s\n"
       "fmla z17.s, p0/M, z0.s, z6.s\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "fmla z18.s, p0/M, z1.s, z6.s\n"
       "fmla z19.s, p0/M, z2.s, z6.s\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
-      "addvl x20, x20, #3\n"
       "fmla z20.s, p0/M, z0.s, z3.s\n"
       "fmla z21.s, p0/M, z1.s, z3.s\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
+      "addvl x22, x22, #3\n"
       "fmla z22.s, p0/M, z2.s, z3.s\n"
       "fmla z23.s, p0/M, z0.s, z4.s\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z24.s, p0/M, z1.s, z4.s\n"
       "fmla z25.s, p0/M, z2.s, z4.s\n"
       "fmla z26.s, p0/M, z0.s, z5.s\n"
@@ -230,7 +230,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "fmla z31.s, p0/M, z2.s, z6.s\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -261,7 +261,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
index e02db6ec48..9d1c0c3728 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -32,42 +32,44 @@ void sve_interleaved_fp32_mla_8x3VL(
     float *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const float *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/1) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "mov %x[Apanel], x21\n"
+      "cmp x20, #0x2\n"
       "mov z8.b, #0x0\n"
       "mov z9.b, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
       "mov z10.b, #0x0\n"
       "mov z11.b, #0x0\n"
-      "ld1w { z4.s }, p0/Z, [x20]\n"
+      "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "mov z12.b, #0x0\n"
       "mov z13.b, #0x0\n"
-      "mov %x[Apanel], x21\n"
+      "ld1w { z4.s }, p0/Z, [x22]\n"
       "mov z14.b, #0x0\n"
       "mov z15.b, #0x0\n"
-      "cmp x19, #0x2\n"
+      "ld1w { z5.s }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.b, #0x0\n"
       "mov z17.b, #0x0\n"
+      "ld1w { z6.s }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z18.b, #0x0\n"
       "mov z19.b, #0x0\n"
-      "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
       "mov z20.b, #0x0\n"
       "mov z21.b, #0x0\n"
-      "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "mov z22.b, #0x0\n"
       "mov z23.b, #0x0\n"
       "mov z24.b, #0x0\n"
@@ -82,31 +84,29 @@ void sve_interleaved_fp32_mla_8x3VL(
       "3:"  // main loop head
       "fmla z8.s, z4.s, z0.s[0]\n"
       "fmla z11.s, z4.s, z0.s[1]\n"
-      "ld1w { z5.s }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1rqw { z2.s }, p0/Z, [%x[Apanel], #32]\n"
       "fmla z14.s, z4.s, z0.s[2]\n"
       "fmla z17.s, z4.s, z0.s[3]\n"
-      "ld1w { z6.s }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #48]\n"
       "fmla z20.s, z4.s, z1.s[0]\n"
       "fmla z23.s, z4.s, z1.s[1]\n"
-      "ld1rqw { z2.s }, p0/Z, [%x[Apanel], #32]\n"
+      "sub x20, x20, #0x2\n"
       "fmla z26.s, z4.s, z1.s[2]\n"
       "fmla z29.s, z4.s, z1.s[3]\n"
-      "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1w { z4.s }, p0/Z, [x22, #3, MUL VL]\n"
       "fmla z9.s, z5.s, z0.s[0]\n"
       "fmla z12.s, z5.s, z0.s[1]\n"
-      "ld1w { z4.s }, p0/Z, [x20, #3, MUL VL]\n"
+      "cmp x20, #0x2\n"
       "fmla z15.s, z5.s, z0.s[2]\n"
       "fmla z18.s, z5.s, z0.s[3]\n"
-      "sub x19, x19, #0x2\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       "fmla z21.s, z5.s, z1.s[0]\n"
       "fmla z24.s, z5.s, z1.s[1]\n"
-      "cmp x19, #0x2\n"
       "fmla z27.s, z5.s, z1.s[2]\n"
       "fmla z30.s, z5.s, z1.s[3]\n"
-      "ld1w { z5.s }, p0/Z, [x20, #4, MUL VL]\n"
+      "ld1w { z5.s }, p0/Z, [x22, #4, MUL VL]\n"
       "fmla z10.s, z6.s, z0.s[0]\n"
       "fmla z13.s, z6.s, z0.s[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       "fmla z16.s, z6.s, z0.s[2]\n"
       "fmla z19.s, z6.s, z0.s[3]\n"
       "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
@@ -114,26 +114,27 @@ void sve_interleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z1.s[1]\n"
       "fmla z28.s, z6.s, z1.s[2]\n"
       "fmla z31.s, z6.s, z1.s[3]\n"
-      "ld1w { z6.s }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1w { z6.s }, p0/Z, [x22, #5, MUL VL]\n"
+      "addvl x22, x22, #6\n"
       "fmla z8.s, z4.s, z2.s[0]\n"
       "fmla z11.s, z4.s, z2.s[1]\n"
       "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "fmla z14.s, z4.s, z2.s[2]\n"
       "fmla z17.s, z4.s, z2.s[3]\n"
-      "addvl x20, x20, #6\n"
       "fmla z20.s, z4.s, z3.s[0]\n"
       "fmla z23.s, z4.s, z3.s[1]\n"
       "fmla z26.s, z4.s, z3.s[2]\n"
       "fmla z29.s, z4.s, z3.s[3]\n"
+      "ld1w { z4.s }, p0/Z, [x22]\n"
       "fmla z9.s, z5.s, z2.s[0]\n"
       "fmla z12.s, z5.s, z2.s[1]\n"
-      "ld1w { z4.s }, p0/Z, [x20]\n"
       "fmla z15.s, z5.s, z2.s[2]\n"
       "fmla z18.s, z5.s, z2.s[3]\n"
       "fmla z21.s, z5.s, z3.s[0]\n"
       "fmla z24.s, z5.s, z3.s[1]\n"
       "fmla z27.s, z5.s, z3.s[2]\n"
       "fmla z30.s, z5.s, z3.s[3]\n"
+      "ld1w { z5.s }, p0/Z, [x22, #1, MUL VL]\n"
       "fmla z10.s, z6.s, z2.s[0]\n"
       "fmla z13.s, z6.s, z2.s[1]\n"
       "fmla z16.s, z6.s, z2.s[2]\n"
@@ -142,20 +143,19 @@ void sve_interleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z3.s[1]\n"
       "fmla z28.s, z6.s, z3.s[2]\n"
       "fmla z31.s, z6.s, z3.s[3]\n"
+      "ld1w { z6.s }, p0/Z, [x22, #2, MUL VL]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "fmla z8.s, z4.s, z0.s[0]\n"
       "fmla z11.s, z4.s, z0.s[1]\n"
-      "ld1w { z5.s }, p0/Z, [x20, #1, MUL VL]\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z14.s, z4.s, z0.s[2]\n"
       "fmla z17.s, z4.s, z0.s[3]\n"
-      "ld1w { z6.s }, p0/Z, [x20, #2, MUL VL]\n"
+      "addvl x22, x22, #3\n"
       "fmla z20.s, z4.s, z1.s[0]\n"
       "fmla z23.s, z4.s, z1.s[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
       "fmla z26.s, z4.s, z1.s[2]\n"
       "fmla z29.s, z4.s, z1.s[3]\n"
-      "addvl x20, x20, #3\n"
       "fmla z9.s, z5.s, z0.s[0]\n"
       "fmla z12.s, z5.s, z0.s[1]\n"
       "fmla z15.s, z5.s, z0.s[2]\n"
@@ -172,19 +172,19 @@ void sve_interleaved_fp32_mla_8x3VL(
       "fmla z25.s, z6.s, z1.s[1]\n"
       "fmla z28.s, z6.s, z1.s[2]\n"
       "fmla z31.s, z6.s, z1.s[3]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n"
       "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ld1w { z7.s }, p0/Z, [x20]\n"
-      "ld1w { z4.s }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1w { z5.s }, p0/Z, [x20, #2, MUL VL]\n"
-      "addvl x20, x20, #3\n"
+      "ld1w { z7.s }, p0/Z, [x22]\n"
+      "ld1w { z4.s }, p0/Z, [x22, #1, MUL VL]\n"
       "fmla z8.s, z7.s, z0.s[0]\n"
+      "ld1w { z5.s }, p0/Z, [x22, #2, MUL VL]\n"
       "fmla z11.s, z7.s, z0.s[1]\n"
       "fmla z14.s, z7.s, z0.s[2]\n"
       "fmla z17.s, z7.s, z0.s[3]\n"
       "fmla z20.s, z7.s, z1.s[0]\n"
+      "addvl x22, x22, #3\n"
       "fmla z23.s, z7.s, z1.s[1]\n"
       "fmla z26.s, z7.s, z1.s[2]\n"
       "fmla z29.s, z7.s, z1.s[3]\n"
@@ -206,7 +206,7 @@ void sve_interleaved_fp32_mla_8x3VL(
       "fmla z31.s, z5.s, z1.s[3]\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -237,7 +237,7 @@ void sve_interleaved_fp32_mla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp
index 5ca4b73b8a..a7ca48d87a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,34 +33,34 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
     int32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const int8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z12.s, #0x0\n"
       "mov z13.s, #0x0\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
@@ -87,7 +87,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "3:"  // main loop head
       "sdot z8.s, z0.b, z3.b\n"
       "sdot z9.s, z1.b, z3.b\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "sdot z10.s, z2.b, z3.b\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "sdot z11.s, z0.b, z4.b\n"
@@ -96,7 +96,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "sdot z14.s, z0.b, z5.b\n"
       "sdot z15.s, z1.b, z5.b\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "sdot z16.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "sdot z17.s, z0.b, z6.b\n"
@@ -116,11 +116,11 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "sdot z28.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n"
       "sdot z29.s, z0.b, z6.b\n"
-      "ld1b { z0.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z0.b }, p0/Z, [x22, #3, MUL VL]\n"
       "sdot z30.s, z1.b, z6.b\n"
       "sdot z31.s, z2.b, z6.b\n"
-      "ld1b { z1.b }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n"
       "sdot z8.s, z0.b, z3.b\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n"
       "sdot z9.s, z1.b, z3.b\n"
@@ -132,7 +132,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n"
       "sdot z14.s, z0.b, z5.b\n"
       "sdot z15.s, z1.b, z5.b\n"
-      "addvl x20, x20, #6\n"
+      "addvl x22, x22, #6\n"
       "sdot z16.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n"
       "sdot z17.s, z0.b, z6.b\n"
@@ -152,18 +152,18 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "sdot z27.s, z1.b, z5.b\n"
       "sdot z28.s, z2.b, z5.b\n"
       "sdot z29.s, z0.b, z6.b\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
       "sdot z30.s, z1.b, z6.b\n"
       "sdot z31.s, z2.b, z6.b\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "sdot z8.s, z0.b, z3.b\n"
       "sdot z9.s, z1.b, z3.b\n"
-      "addvl x20, x20, #3\n"
+      "addvl x22, x22, #3\n"
       "sdot z10.s, z2.b, z3.b\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "sdot z11.s, z0.b, z4.b\n"
@@ -191,10 +191,10 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "sdot z29.s, z0.b, z6.b\n"
       "sdot z30.s, z1.b, z6.b\n"
       "sdot z31.s, z2.b, z6.b\n"
-      "cbz x19, 5f\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "cbz x20, 5f\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
       "sdot z8.s, z0.b, z3.b\n"
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
@@ -203,24 +203,24 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "sdot z10.s, z2.b, z3.b\n"
       "sdot z11.s, z0.b, z4.b\n"
-      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "sdot z12.s, z1.b, z4.b\n"
       "sdot z13.s, z2.b, z4.b\n"
-      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
+      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "sdot z14.s, z0.b, z5.b\n"
       "sdot z15.s, z1.b, z5.b\n"
+      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "sdot z16.s, z2.b, z5.b\n"
       "sdot z17.s, z0.b, z6.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "sdot z18.s, z1.b, z6.b\n"
       "sdot z19.s, z2.b, z6.b\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
-      "addvl x20, x20, #3\n"
       "sdot z20.s, z0.b, z3.b\n"
       "sdot z21.s, z1.b, z3.b\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
+      "addvl x22, x22, #3\n"
       "sdot z22.s, z2.b, z3.b\n"
       "sdot z23.s, z0.b, z4.b\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "sdot z24.s, z1.b, z4.b\n"
       "sdot z25.s, z2.b, z4.b\n"
       "sdot z26.s, z0.b, z5.b\n"
@@ -231,7 +231,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "sdot z31.s, z2.b, z6.b\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -262,7 +262,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
index 5fb938b20f..e5f59d220b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,42 +33,44 @@ void sve_interleaved_s8s32_dot_8x3VL(
     int32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const int8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "mov %x[Apanel], x21\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
+      "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z12.s, #0x0\n"
       "mov z13.s, #0x0\n"
-      "mov %x[Apanel], x21\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
-      "cmp x19, #0x2\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.s, #0x0\n"
       "mov z17.s, #0x0\n"
+      "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
-      "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "mov z20.s, #0x0\n"
       "mov z21.s, #0x0\n"
-      "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "mov z24.s, #0x0\n"
@@ -83,31 +85,29 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "3:"  // main loop head
       "sdot z8.s, z4.b, z0.b[0]\n"
       "sdot z11.s, z4.b, z0.b[1]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "sdot z14.s, z4.b, z0.b[2]\n"
       "sdot z17.s, z4.b, z0.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
       "sdot z23.s, z4.b, z1.b[1]\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
+      "sub x20, x20, #0x2\n"
       "sdot z26.s, z4.b, z1.b[2]\n"
       "sdot z29.s, z4.b, z1.b[3]\n"
-      "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n"
       "sdot z9.s, z5.b, z0.b[0]\n"
       "sdot z12.s, z5.b, z0.b[1]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "cmp x20, #0x2\n"
       "sdot z15.s, z5.b, z0.b[2]\n"
       "sdot z18.s, z5.b, z0.b[3]\n"
-      "sub x19, x19, #0x2\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       "sdot z21.s, z5.b, z1.b[0]\n"
       "sdot z24.s, z5.b, z1.b[1]\n"
-      "cmp x19, #0x2\n"
       "sdot z27.s, z5.b, z1.b[2]\n"
       "sdot z30.s, z5.b, z1.b[3]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #4, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #4, MUL VL]\n"
       "sdot z10.s, z6.b, z0.b[0]\n"
       "sdot z13.s, z6.b, z0.b[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       "sdot z16.s, z6.b, z0.b[2]\n"
       "sdot z19.s, z6.b, z0.b[3]\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
@@ -115,26 +115,27 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "sdot z25.s, z6.b, z1.b[1]\n"
       "sdot z28.s, z6.b, z1.b[2]\n"
       "sdot z31.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #5, MUL VL]\n"
+      "addvl x22, x22, #6\n"
       "sdot z8.s, z4.b, z2.b[0]\n"
       "sdot z11.s, z4.b, z2.b[1]\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "sdot z14.s, z4.b, z2.b[2]\n"
       "sdot z17.s, z4.b, z2.b[3]\n"
-      "addvl x20, x20, #6\n"
       "sdot z20.s, z4.b, z3.b[0]\n"
       "sdot z23.s, z4.b, z3.b[1]\n"
       "sdot z26.s, z4.b, z3.b[2]\n"
       "sdot z29.s, z4.b, z3.b[3]\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "sdot z9.s, z5.b, z2.b[0]\n"
       "sdot z12.s, z5.b, z2.b[1]\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
       "sdot z15.s, z5.b, z2.b[2]\n"
       "sdot z18.s, z5.b, z2.b[3]\n"
       "sdot z21.s, z5.b, z3.b[0]\n"
       "sdot z24.s, z5.b, z3.b[1]\n"
       "sdot z27.s, z5.b, z3.b[2]\n"
       "sdot z30.s, z5.b, z3.b[3]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "sdot z10.s, z6.b, z2.b[0]\n"
       "sdot z13.s, z6.b, z2.b[1]\n"
       "sdot z16.s, z6.b, z2.b[2]\n"
@@ -143,20 +144,19 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "sdot z25.s, z6.b, z3.b[1]\n"
       "sdot z28.s, z6.b, z3.b[2]\n"
       "sdot z31.s, z6.b, z3.b[3]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "sdot z8.s, z4.b, z0.b[0]\n"
       "sdot z11.s, z4.b, z0.b[1]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "sdot z14.s, z4.b, z0.b[2]\n"
       "sdot z17.s, z4.b, z0.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "addvl x22, x22, #3\n"
       "sdot z20.s, z4.b, z1.b[0]\n"
       "sdot z23.s, z4.b, z1.b[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
       "sdot z26.s, z4.b, z1.b[2]\n"
       "sdot z29.s, z4.b, z1.b[3]\n"
-      "addvl x20, x20, #3\n"
       "sdot z9.s, z5.b, z0.b[0]\n"
       "sdot z12.s, z5.b, z0.b[1]\n"
       "sdot z15.s, z5.b, z0.b[2]\n"
@@ -173,19 +173,19 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "sdot z25.s, z6.b, z1.b[1]\n"
       "sdot z28.s, z6.b, z1.b[2]\n"
       "sdot z31.s, z6.b, z1.b[3]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ld1b { z7.b }, p0/Z, [x20]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "addvl x20, x20, #3\n"
+      "ld1b { z7.b }, p0/Z, [x22]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #1, MUL VL]\n"
       "sdot z8.s, z7.b, z0.b[0]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n"
       "sdot z11.s, z7.b, z0.b[1]\n"
       "sdot z14.s, z7.b, z0.b[2]\n"
       "sdot z17.s, z7.b, z0.b[3]\n"
       "sdot z20.s, z7.b, z1.b[0]\n"
+      "addvl x22, x22, #3\n"
       "sdot z23.s, z7.b, z1.b[1]\n"
       "sdot z26.s, z7.b, z1.b[2]\n"
       "sdot z29.s, z7.b, z1.b[3]\n"
@@ -207,7 +207,7 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "sdot z31.s, z5.b, z1.b[3]\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -238,7 +238,7 @@ void sve_interleaved_s8s32_dot_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
index b8f1864af3..104d5f918e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,28 +33,28 @@ void sve_interleaved_s8s32_mmla_8x3VL(
     int32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const int8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
@@ -63,13 +63,13 @@ void sve_interleaved_s8s32_mmla_8x3VL(
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.s, #0x0\n"
       "mov z17.s, #0x0\n"
       "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
-      "addvl x20, x20, #2\n"
+      "addvl x22, x22, #2\n"
       "mov z20.s, #0x0\n"
       "mov z21.s, #0x0\n"
       "add %x[Apanel], %x[Apanel], #0x30\n"
@@ -87,143 +87,143 @@ void sve_interleaved_s8s32_mmla_8x3VL(
       "3:"  // main loop head
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45049808  // smmla z8.s, z0.b, z4.b\n"
-      ".inst 0x4504982e  // smmla z14.s, z1.b, z4.b\n"
       ".inst 0x4505980b  // smmla z11.s, z0.b, z5.b\n"
+      ".inst 0x4504982e  // smmla z14.s, z1.b, z4.b\n"
       ".inst 0x45059831  // smmla z17.s, z1.b, z5.b\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       ".inst 0x45049854  // smmla z20.s, z2.b, z4.b\n"
       ".inst 0x45059857  // smmla z23.s, z2.b, z5.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x4504987a  // smmla z26.s, z3.b, z4.b\n"
       ".inst 0x4505987d  // smmla z29.s, z3.b, z5.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x45069809  // smmla z9.s, z0.b, z6.b\n"
+      ".inst 0x4507980c  // smmla z12.s, z0.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      ".inst 0x45079832  // smmla z18.s, z1.b, z7.b\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      ".inst 0x45079858  // smmla z24.s, z2.b, z7.b\n"
+      "cmp x20, #0x2\n"
       ".inst 0x4506987b  // smmla z27.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #4, MUL VL]\n"
-      ".inst 0x4507980c  // smmla z12.s, z0.b, z7.b\n"
+      ".inst 0x4507987e  // smmla z30.s, z3.b, z7.b\n"
+      "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n"
       ".inst 0x4504980a  // smmla z10.s, z0.b, z4.b\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x4505980d  // smmla z13.s, z0.b, z5.b\n"
-      ".inst 0x45079832  // smmla z18.s, z1.b, z7.b\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #16]\n"
       ".inst 0x45049830  // smmla z16.s, z1.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #32]\n"
-      ".inst 0x45079858  // smmla z24.s, z2.b, z7.b\n"
-      ".inst 0x4507987e  // smmla z30.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #5, MUL VL]\n"
       ".inst 0x45049856  // smmla z22.s, z2.b, z4.b\n"
       ".inst 0x45059859  // smmla z25.s, z2.b, z5.b\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x4504987c  // smmla z28.s, z3.b, z4.b\n"
       ".inst 0x4505987f  // smmla z31.s, z3.b, z5.b\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n"
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #64]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #6, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #7, MUL VL]\n"
-      "addvl x20, x20, #16\n"
+      "ld1b { z4.b }, p0/Z, [x22, #6, MUL VL]\n"
       ".inst 0x45069808  // smmla z8.s, z0.b, z6.b\n"
-      ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
-      "cmp x19, #0x2\n"
+      "ld1b { z5.b }, p0/Z, [x22, #7, MUL VL]\n"
+      "addvl x22, x22, #16\n"
       ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45079831  // smmla z17.s, z1.b, z7.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
       ".inst 0x45079857  // smmla z23.s, z2.b, z7.b\n"
       ".inst 0x4506987a  // smmla z26.s, z3.b, z6.b\n"
       ".inst 0x4507987d  // smmla z29.s, z3.b, z7.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #-8, MUL VL]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #-7, MUL VL]\n"
       ".inst 0x45049809  // smmla z9.s, z0.b, z4.b\n"
+      ".inst 0x4505980c  // smmla z12.s, z0.b, z5.b\n"
       ".inst 0x4504982f  // smmla z15.s, z1.b, z4.b\n"
+      ".inst 0x45059832  // smmla z18.s, z1.b, z5.b\n"
       ".inst 0x45049855  // smmla z21.s, z2.b, z4.b\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
       ".inst 0x4504987b  // smmla z27.s, z3.b, z4.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #-6, MUL VL]\n"
-      ".inst 0x4505980c  // smmla z12.s, z0.b, z5.b\n"
+      ".inst 0x4505987e  // smmla z30.s, z3.b, z5.b\n"
+      "ld1b { z4.b }, p0/Z, [x22, #-6, MUL VL]\n"
       ".inst 0x4506980a  // smmla z10.s, z0.b, z6.b\n"
       ".inst 0x4507980d  // smmla z13.s, z0.b, z7.b\n"
-      ".inst 0x45059832  // smmla z18.s, z1.b, z5.b\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #80]\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
       ".inst 0x45079833  // smmla z19.s, z1.b, z7.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #96]\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
-      ".inst 0x4505987e  // smmla z30.s, z3.b, z5.b\n"
-      "ld1b { z5.b }, p0/Z, [x20, #-5, MUL VL]\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #-5, MUL VL]\n"
       ".inst 0x4506987c  // smmla z28.s, z3.b, z6.b\n"
       ".inst 0x4507987f  // smmla z31.s, z3.b, z7.b\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "addvl x20, x20, #-4\n"
+      "addvl x22, x22, #-4\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45049808  // smmla z8.s, z0.b, z4.b\n"
-      ".inst 0x4504982e  // smmla z14.s, z1.b, z4.b\n"
       ".inst 0x4505980b  // smmla z11.s, z0.b, z5.b\n"
+      ".inst 0x4504982e  // smmla z14.s, z1.b, z4.b\n"
       ".inst 0x45059831  // smmla z17.s, z1.b, z5.b\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       ".inst 0x45049854  // smmla z20.s, z2.b, z4.b\n"
       ".inst 0x45059857  // smmla z23.s, z2.b, z5.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x4504987a  // smmla z26.s, z3.b, z4.b\n"
       ".inst 0x4505987d  // smmla z29.s, z3.b, z5.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x45069809  // smmla z9.s, z0.b, z6.b\n"
+      ".inst 0x4507980c  // smmla z12.s, z0.b, z7.b\n"
       ".inst 0x4506982f  // smmla z15.s, z1.b, z6.b\n"
+      ".inst 0x45079832  // smmla z18.s, z1.b, z7.b\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x45069855  // smmla z21.s, z2.b, z6.b\n"
+      ".inst 0x45079858  // smmla z24.s, z2.b, z7.b\n"
+      "addvl x22, x22, #4\n"
       ".inst 0x4506987b  // smmla z27.s, z3.b, z6.b\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x4507980c  // smmla z12.s, z0.b, z7.b\n"
+      ".inst 0x4507987e  // smmla z30.s, z3.b, z7.b\n"
       ".inst 0x4504980a  // smmla z10.s, z0.b, z4.b\n"
-      "addvl x20, x20, #4\n"
       ".inst 0x4505980d  // smmla z13.s, z0.b, z5.b\n"
-      ".inst 0x45079832  // smmla z18.s, z1.b, z7.b\n"
       ".inst 0x45049830  // smmla z16.s, z1.b, z4.b\n"
       ".inst 0x45059833  // smmla z19.s, z1.b, z5.b\n"
-      ".inst 0x45079858  // smmla z24.s, z2.b, z7.b\n"
-      ".inst 0x4507987e  // smmla z30.s, z3.b, z7.b\n"
       ".inst 0x45049856  // smmla z22.s, z2.b, z4.b\n"
       ".inst 0x45059859  // smmla z25.s, z2.b, z5.b\n"
       ".inst 0x4504987c  // smmla z28.s, z3.b, z4.b\n"
       ".inst 0x4505987f  // smmla z31.s, z3.b, z5.b\n"
-      "cbz x19, 5f\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "cbz x20, 5f\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45069808  // smmla z8.s, z0.b, z6.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
-      ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
+      ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
       "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
-      ".inst 0x4507980b  // smmla z11.s, z0.b, z7.b\n"
+      ".inst 0x4506982e  // smmla z14.s, z1.b, z6.b\n"
       ".inst 0x45079831  // smmla z17.s, z1.b, z7.b\n"
       ".inst 0x45069854  // smmla z20.s, z2.b, z6.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
       ".inst 0x45079857  // smmla z23.s, z2.b, z7.b\n"
       ".inst 0x4506987a  // smmla z26.s, z3.b, z6.b\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x4507987d  // smmla z29.s, z3.b, z7.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x45049809  // smmla z9.s, z0.b, z4.b\n"
+      ".inst 0x4505980c  // smmla z12.s, z0.b, z5.b\n"
+      "addvl x22, x22, #6\n"
       ".inst 0x4504982f  // smmla z15.s, z1.b, z4.b\n"
+      ".inst 0x45059832  // smmla z18.s, z1.b, z5.b\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x45049855  // smmla z21.s, z2.b, z4.b\n"
+      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
       ".inst 0x4504987b  // smmla z27.s, z3.b, z4.b\n"
-      "addvl x20, x20, #6\n"
-      ".inst 0x4505980c  // smmla z12.s, z0.b, z5.b\n"
+      ".inst 0x4505987e  // smmla z30.s, z3.b, z5.b\n"
       ".inst 0x4506980a  // smmla z10.s, z0.b, z6.b\n"
       ".inst 0x4507980d  // smmla z13.s, z0.b, z7.b\n"
-      ".inst 0x45059832  // smmla z18.s, z1.b, z5.b\n"
       ".inst 0x45069830  // smmla z16.s, z1.b, z6.b\n"
       ".inst 0x45079833  // smmla z19.s, z1.b, z7.b\n"
-      ".inst 0x45059858  // smmla z24.s, z2.b, z5.b\n"
-      ".inst 0x4505987e  // smmla z30.s, z3.b, z5.b\n"
       ".inst 0x45069856  // smmla z22.s, z2.b, z6.b\n"
       ".inst 0x45079859  // smmla z25.s, z2.b, z7.b\n"
       ".inst 0x4506987c  // smmla z28.s, z3.b, z6.b\n"
@@ -243,7 +243,7 @@ void sve_interleaved_s8s32_mmla_8x3VL(
       "uzp2 z14.d, z14.d, z17.d\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
       "uzp1 z17.d, z15.d, z18.d\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n"
       "uzp2 z15.d, z15.d, z18.d\n"
       "uzp1 z18.d, z16.d, z19.d\n"
@@ -285,7 +285,7 @@ void sve_interleaved_s8s32_mmla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp
index 1e2fb138fd..2bfec8f350 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,34 +33,34 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
     uint32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const uint8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z12.s, #0x0\n"
       "mov z13.s, #0x0\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
@@ -87,7 +87,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "3:"  // main loop head
       "udot z8.s, z0.b, z3.b\n"
       "udot z9.s, z1.b, z3.b\n"
-      "sub x19, x19, #0x2\n"
+      "sub x20, x20, #0x2\n"
       "udot z10.s, z2.b, z3.b\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "udot z11.s, z0.b, z4.b\n"
@@ -96,7 +96,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "udot z14.s, z0.b, z5.b\n"
       "udot z15.s, z1.b, z5.b\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "udot z16.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "udot z17.s, z0.b, z6.b\n"
@@ -116,11 +116,11 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "udot z28.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n"
       "udot z29.s, z0.b, z6.b\n"
-      "ld1b { z0.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z0.b }, p0/Z, [x22, #3, MUL VL]\n"
       "udot z30.s, z1.b, z6.b\n"
       "udot z31.s, z2.b, z6.b\n"
-      "ld1b { z1.b }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n"
       "udot z8.s, z0.b, z3.b\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n"
       "udot z9.s, z1.b, z3.b\n"
@@ -132,7 +132,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n"
       "udot z14.s, z0.b, z5.b\n"
       "udot z15.s, z1.b, z5.b\n"
-      "addvl x20, x20, #6\n"
+      "addvl x22, x22, #6\n"
       "udot z16.s, z2.b, z5.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n"
       "udot z17.s, z0.b, z6.b\n"
@@ -152,18 +152,18 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "udot z27.s, z1.b, z5.b\n"
       "udot z28.s, z2.b, z5.b\n"
       "udot z29.s, z0.b, z6.b\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
       "udot z30.s, z1.b, z6.b\n"
       "udot z31.s, z2.b, z6.b\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "udot z8.s, z0.b, z3.b\n"
       "udot z9.s, z1.b, z3.b\n"
-      "addvl x20, x20, #3\n"
+      "addvl x22, x22, #3\n"
       "udot z10.s, z2.b, z3.b\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "udot z11.s, z0.b, z4.b\n"
@@ -191,10 +191,10 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "udot z29.s, z0.b, z6.b\n"
       "udot z30.s, z1.b, z6.b\n"
       "udot z31.s, z2.b, z6.b\n"
-      "cbz x19, 5f\n"
-      "ld1b { z0.b }, p0/Z, [x20]\n"
-      "ld1b { z1.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z2.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "cbz x20, 5f\n"
+      "ld1b { z0.b }, p0/Z, [x22]\n"
+      "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n"
+      "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n"
       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
       "udot z8.s, z0.b, z3.b\n"
       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
@@ -203,24 +203,24 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
       "udot z10.s, z2.b, z3.b\n"
       "udot z11.s, z0.b, z4.b\n"
-      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "udot z12.s, z1.b, z4.b\n"
       "udot z13.s, z2.b, z4.b\n"
-      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
+      "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
       "udot z14.s, z0.b, z5.b\n"
       "udot z15.s, z1.b, z5.b\n"
+      "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
       "udot z16.s, z2.b, z5.b\n"
       "udot z17.s, z0.b, z6.b\n"
       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
       "udot z18.s, z1.b, z6.b\n"
       "udot z19.s, z2.b, z6.b\n"
       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
-      "addvl x20, x20, #3\n"
       "udot z20.s, z0.b, z3.b\n"
       "udot z21.s, z1.b, z3.b\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
+      "addvl x22, x22, #3\n"
       "udot z22.s, z2.b, z3.b\n"
       "udot z23.s, z0.b, z4.b\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "udot z24.s, z1.b, z4.b\n"
       "udot z25.s, z2.b, z4.b\n"
       "udot z26.s, z0.b, z5.b\n"
@@ -231,7 +231,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "udot z31.s, z2.b, z6.b\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -262,7 +262,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
index f1642d0b21..99fff4e83d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,16 +10,16 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 #ifdef ARM_COMPUTE_ENABLE_SVE
 
@@ -33,42 +33,44 @@ void sve_interleaved_u8u32_dot_8x3VL(
     uint32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const uint8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/4) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
+      "mov %x[Apanel], x21\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
+      "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z12.s, #0x0\n"
       "mov z13.s, #0x0\n"
-      "mov %x[Apanel], x21\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
-      "cmp x19, #0x2\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.s, #0x0\n"
       "mov z17.s, #0x0\n"
+      "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
-      "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "mov z20.s, #0x0\n"
       "mov z21.s, #0x0\n"
-      "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z22.s, #0x0\n"
       "mov z23.s, #0x0\n"
       "mov z24.s, #0x0\n"
@@ -83,31 +85,29 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "3:"  // main loop head
       "udot z8.s, z4.b, z0.b[0]\n"
       "udot z11.s, z4.b, z0.b[1]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "udot z14.s, z4.b, z0.b[2]\n"
       "udot z17.s, z4.b, z0.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
       "udot z20.s, z4.b, z1.b[0]\n"
       "udot z23.s, z4.b, z1.b[1]\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
+      "sub x20, x20, #0x2\n"
       "udot z26.s, z4.b, z1.b[2]\n"
       "udot z29.s, z4.b, z1.b[3]\n"
-      "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n"
       "udot z9.s, z5.b, z0.b[0]\n"
       "udot z12.s, z5.b, z0.b[1]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "cmp x20, #0x2\n"
       "udot z15.s, z5.b, z0.b[2]\n"
       "udot z18.s, z5.b, z0.b[3]\n"
-      "sub x19, x19, #0x2\n"
+      "add %x[Apanel], %x[Apanel], #0x40\n"
       "udot z21.s, z5.b, z1.b[0]\n"
       "udot z24.s, z5.b, z1.b[1]\n"
-      "cmp x19, #0x2\n"
       "udot z27.s, z5.b, z1.b[2]\n"
       "udot z30.s, z5.b, z1.b[3]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #4, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #4, MUL VL]\n"
       "udot z10.s, z6.b, z0.b[0]\n"
       "udot z13.s, z6.b, z0.b[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x40\n"
       "udot z16.s, z6.b, z0.b[2]\n"
       "udot z19.s, z6.b, z0.b[3]\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
@@ -115,26 +115,27 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "udot z25.s, z6.b, z1.b[1]\n"
       "udot z28.s, z6.b, z1.b[2]\n"
       "udot z31.s, z6.b, z1.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #5, MUL VL]\n"
+      "addvl x22, x22, #6\n"
       "udot z8.s, z4.b, z2.b[0]\n"
       "udot z11.s, z4.b, z2.b[1]\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "udot z14.s, z4.b, z2.b[2]\n"
       "udot z17.s, z4.b, z2.b[3]\n"
-      "addvl x20, x20, #6\n"
       "udot z20.s, z4.b, z3.b[0]\n"
       "udot z23.s, z4.b, z3.b[1]\n"
       "udot z26.s, z4.b, z3.b[2]\n"
       "udot z29.s, z4.b, z3.b[3]\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "udot z9.s, z5.b, z2.b[0]\n"
       "udot z12.s, z5.b, z2.b[1]\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
       "udot z15.s, z5.b, z2.b[2]\n"
       "udot z18.s, z5.b, z2.b[3]\n"
       "udot z21.s, z5.b, z3.b[0]\n"
       "udot z24.s, z5.b, z3.b[1]\n"
       "udot z27.s, z5.b, z3.b[2]\n"
       "udot z30.s, z5.b, z3.b[3]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "udot z10.s, z6.b, z2.b[0]\n"
       "udot z13.s, z6.b, z2.b[1]\n"
       "udot z16.s, z6.b, z2.b[2]\n"
@@ -143,20 +144,19 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "udot z25.s, z6.b, z3.b[1]\n"
       "udot z28.s, z6.b, z3.b[2]\n"
       "udot z31.s, z6.b, z3.b[3]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "udot z8.s, z4.b, z0.b[0]\n"
       "udot z11.s, z4.b, z0.b[1]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "add %x[Apanel], %x[Apanel], #0x20\n"
       "udot z14.s, z4.b, z0.b[2]\n"
       "udot z17.s, z4.b, z0.b[3]\n"
-      "ld1b { z6.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "addvl x22, x22, #3\n"
       "udot z20.s, z4.b, z1.b[0]\n"
       "udot z23.s, z4.b, z1.b[1]\n"
-      "add %x[Apanel], %x[Apanel], #0x20\n"
       "udot z26.s, z4.b, z1.b[2]\n"
       "udot z29.s, z4.b, z1.b[3]\n"
-      "addvl x20, x20, #3\n"
       "udot z9.s, z5.b, z0.b[0]\n"
       "udot z12.s, z5.b, z0.b[1]\n"
       "udot z15.s, z5.b, z0.b[2]\n"
@@ -173,19 +173,19 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "udot z25.s, z6.b, z1.b[1]\n"
       "udot z28.s, z6.b, z1.b[2]\n"
       "udot z31.s, z6.b, z1.b[3]\n"
-      "cbz x19, 5f\n"
+      "cbz x20, 5f\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ld1b { z7.b }, p0/Z, [x20]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #1, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "addvl x20, x20, #3\n"
+      "ld1b { z7.b }, p0/Z, [x22]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #1, MUL VL]\n"
       "udot z8.s, z7.b, z0.b[0]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n"
       "udot z11.s, z7.b, z0.b[1]\n"
       "udot z14.s, z7.b, z0.b[2]\n"
       "udot z17.s, z7.b, z0.b[3]\n"
       "udot z20.s, z7.b, z1.b[0]\n"
+      "addvl x22, x22, #3\n"
       "udot z23.s, z7.b, z1.b[1]\n"
       "udot z26.s, z7.b, z1.b[2]\n"
       "udot z29.s, z7.b, z1.b[3]\n"
@@ -207,7 +207,7 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "udot z31.s, z5.b, z1.b[3]\n"
       "5:"  // multiply loop done
       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
@@ -238,7 +238,7 @@ void sve_interleaved_u8u32_dot_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
index c4fdfa6abc..0b70d034dd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -33,28 +33,28 @@ void sve_interleaved_u8u32_mmla_8x3VL(
     uint32_t *Cpanel, int ablocks, int bblocks, int K) {
 
     struct KernelArgs {
-        size_t bblocks = {};
         size_t K = {};
         const uint8_t *Bpanel = {};
+        size_t bblocks = {};
     } ka;
 
-    ka.bblocks = bblocks;
     ka.K = (K/8) - 1;
     ka.Bpanel = Bpanel;
+    ka.bblocks = bblocks;
 
     __asm__ __volatile__(
       "ptrue p0.b\n"
       "1:"  // Height loop
-      "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
+      "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "mov x21, %x[Apanel]\n"
-      "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
       "2:"  // Width loop
-      "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
+      "ldr x20, [%x[args_ptr], %[offsetof_K]]\n"
       "mov %x[Apanel], x21\n"
-      "cmp x19, #0x2\n"
+      "cmp x20, #0x2\n"
       "mov z8.s, #0x0\n"
       "mov z9.s, #0x0\n"
-      "ld1b { z4.b }, p0/Z, [x20]\n"
+      "ld1b { z4.b }, p0/Z, [x22]\n"
       "mov z10.s, #0x0\n"
       "mov z11.s, #0x0\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
@@ -63,13 +63,13 @@ void sve_interleaved_u8u32_mmla_8x3VL(
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
       "mov z14.s, #0x0\n"
       "mov z15.s, #0x0\n"
-      "ld1b { z5.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n"
       "mov z16.s, #0x0\n"
       "mov z17.s, #0x0\n"
       "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "mov z18.s, #0x0\n"
       "mov z19.s, #0x0\n"
-      "addvl x20, x20, #2\n"
+      "addvl x22, x22, #2\n"
       "mov z20.s, #0x0\n"
       "mov z21.s, #0x0\n"
       "add %x[Apanel], %x[Apanel], #0x30\n"
@@ -87,143 +87,143 @@ void sve_interleaved_u8u32_mmla_8x3VL(
       "3:"  // main loop head
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45c49808  // ummla z8.s, z0.b, z4.b\n"
-      ".inst 0x45c4982e  // ummla z14.s, z1.b, z4.b\n"
       ".inst 0x45c5980b  // ummla z11.s, z0.b, z5.b\n"
+      ".inst 0x45c4982e  // ummla z14.s, z1.b, z4.b\n"
       ".inst 0x45c59831  // ummla z17.s, z1.b, z5.b\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       ".inst 0x45c49854  // ummla z20.s, z2.b, z4.b\n"
       ".inst 0x45c59857  // ummla z23.s, z2.b, z5.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x45c4987a  // ummla z26.s, z3.b, z4.b\n"
       ".inst 0x45c5987d  // ummla z29.s, z3.b, z5.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x45c69809  // ummla z9.s, z0.b, z6.b\n"
+      ".inst 0x45c7980c  // ummla z12.s, z0.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      ".inst 0x45c79832  // ummla z18.s, z1.b, z7.b\n"
+      "sub x20, x20, #0x2\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
+      ".inst 0x45c79858  // ummla z24.s, z2.b, z7.b\n"
+      "cmp x20, #0x2\n"
       ".inst 0x45c6987b  // ummla z27.s, z3.b, z6.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #4, MUL VL]\n"
-      ".inst 0x45c7980c  // ummla z12.s, z0.b, z7.b\n"
+      ".inst 0x45c7987e  // ummla z30.s, z3.b, z7.b\n"
+      "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n"
       ".inst 0x45c4980a  // ummla z10.s, z0.b, z4.b\n"
-      "sub x19, x19, #0x2\n"
       ".inst 0x45c5980d  // ummla z13.s, z0.b, z5.b\n"
-      ".inst 0x45c79832  // ummla z18.s, z1.b, z7.b\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #16]\n"
       ".inst 0x45c49830  // ummla z16.s, z1.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #32]\n"
-      ".inst 0x45c79858  // ummla z24.s, z2.b, z7.b\n"
-      ".inst 0x45c7987e  // ummla z30.s, z3.b, z7.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #5, MUL VL]\n"
       ".inst 0x45c49856  // ummla z22.s, z2.b, z4.b\n"
       ".inst 0x45c59859  // ummla z25.s, z2.b, z5.b\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x45c4987c  // ummla z28.s, z3.b, z4.b\n"
       ".inst 0x45c5987f  // ummla z31.s, z3.b, z5.b\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n"
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #64]\n"
-      "ld1b { z4.b }, p0/Z, [x20, #6, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #7, MUL VL]\n"
-      "addvl x20, x20, #16\n"
+      "ld1b { z4.b }, p0/Z, [x22, #6, MUL VL]\n"
       ".inst 0x45c69808  // ummla z8.s, z0.b, z6.b\n"
-      ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
-      "cmp x19, #0x2\n"
+      "ld1b { z5.b }, p0/Z, [x22, #7, MUL VL]\n"
+      "addvl x22, x22, #16\n"
       ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
+      ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c79831  // ummla z17.s, z1.b, z7.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
       ".inst 0x45c79857  // ummla z23.s, z2.b, z7.b\n"
       ".inst 0x45c6987a  // ummla z26.s, z3.b, z6.b\n"
       ".inst 0x45c7987d  // ummla z29.s, z3.b, z7.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #-8, MUL VL]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #-7, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #-8, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #-7, MUL VL]\n"
       ".inst 0x45c49809  // ummla z9.s, z0.b, z4.b\n"
+      ".inst 0x45c5980c  // ummla z12.s, z0.b, z5.b\n"
       ".inst 0x45c4982f  // ummla z15.s, z1.b, z4.b\n"
+      ".inst 0x45c59832  // ummla z18.s, z1.b, z5.b\n"
       ".inst 0x45c49855  // ummla z21.s, z2.b, z4.b\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
       ".inst 0x45c4987b  // ummla z27.s, z3.b, z4.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #-6, MUL VL]\n"
-      ".inst 0x45c5980c  // ummla z12.s, z0.b, z5.b\n"
+      ".inst 0x45c5987e  // ummla z30.s, z3.b, z5.b\n"
+      "ld1b { z4.b }, p0/Z, [x22, #-6, MUL VL]\n"
       ".inst 0x45c6980a  // ummla z10.s, z0.b, z6.b\n"
       ".inst 0x45c7980d  // ummla z13.s, z0.b, z7.b\n"
-      ".inst 0x45c59832  // ummla z18.s, z1.b, z5.b\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #80]\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
       ".inst 0x45c79833  // ummla z19.s, z1.b, z7.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #96]\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
-      ".inst 0x45c5987e  // ummla z30.s, z3.b, z5.b\n"
-      "ld1b { z5.b }, p0/Z, [x20, #-5, MUL VL]\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
-      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #-5, MUL VL]\n"
       ".inst 0x45c6987c  // ummla z28.s, z3.b, z6.b\n"
       ".inst 0x45c7987f  // ummla z31.s, z3.b, z7.b\n"
+      "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n"
       "add %x[Apanel], %x[Apanel], #0x80\n"
-      "addvl x20, x20, #-4\n"
+      "addvl x22, x22, #-4\n"
       "bge 3b\n"
       "4:"  // main loop skip
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45c49808  // ummla z8.s, z0.b, z4.b\n"
-      ".inst 0x45c4982e  // ummla z14.s, z1.b, z4.b\n"
       ".inst 0x45c5980b  // ummla z11.s, z0.b, z5.b\n"
+      ".inst 0x45c4982e  // ummla z14.s, z1.b, z4.b\n"
       ".inst 0x45c59831  // ummla z17.s, z1.b, z5.b\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       ".inst 0x45c49854  // ummla z20.s, z2.b, z4.b\n"
       ".inst 0x45c59857  // ummla z23.s, z2.b, z5.b\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
       ".inst 0x45c4987a  // ummla z26.s, z3.b, z4.b\n"
       ".inst 0x45c5987d  // ummla z29.s, z3.b, z5.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x45c69809  // ummla z9.s, z0.b, z6.b\n"
+      ".inst 0x45c7980c  // ummla z12.s, z0.b, z7.b\n"
       ".inst 0x45c6982f  // ummla z15.s, z1.b, z6.b\n"
+      ".inst 0x45c79832  // ummla z18.s, z1.b, z7.b\n"
+      "add %x[Apanel], %x[Apanel], #0x10\n"
       ".inst 0x45c69855  // ummla z21.s, z2.b, z6.b\n"
+      ".inst 0x45c79858  // ummla z24.s, z2.b, z7.b\n"
+      "addvl x22, x22, #4\n"
       ".inst 0x45c6987b  // ummla z27.s, z3.b, z6.b\n"
-      "add %x[Apanel], %x[Apanel], #0x10\n"
-      ".inst 0x45c7980c  // ummla z12.s, z0.b, z7.b\n"
+      ".inst 0x45c7987e  // ummla z30.s, z3.b, z7.b\n"
       ".inst 0x45c4980a  // ummla z10.s, z0.b, z4.b\n"
-      "addvl x20, x20, #4\n"
       ".inst 0x45c5980d  // ummla z13.s, z0.b, z5.b\n"
-      ".inst 0x45c79832  // ummla z18.s, z1.b, z7.b\n"
       ".inst 0x45c49830  // ummla z16.s, z1.b, z4.b\n"
       ".inst 0x45c59833  // ummla z19.s, z1.b, z5.b\n"
-      ".inst 0x45c79858  // ummla z24.s, z2.b, z7.b\n"
-      ".inst 0x45c7987e  // ummla z30.s, z3.b, z7.b\n"
       ".inst 0x45c49856  // ummla z22.s, z2.b, z4.b\n"
       ".inst 0x45c59859  // ummla z25.s, z2.b, z5.b\n"
       ".inst 0x45c4987c  // ummla z28.s, z3.b, z4.b\n"
       ".inst 0x45c5987f  // ummla z31.s, z3.b, z5.b\n"
-      "cbz x19, 5f\n"
-      "ld1b { z6.b }, p0/Z, [x20]\n"
+      "cbz x20, 5f\n"
+      "ld1b { z6.b }, p0/Z, [x22]\n"
       "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n"
       ".inst 0x45c69808  // ummla z8.s, z0.b, z6.b\n"
       "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #1, MUL VL]\n"
-      ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
+      "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n"
+      ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
       "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n"
       "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n"
-      ".inst 0x45c7980b  // ummla z11.s, z0.b, z7.b\n"
+      ".inst 0x45c6982e  // ummla z14.s, z1.b, z6.b\n"
       ".inst 0x45c79831  // ummla z17.s, z1.b, z7.b\n"
       ".inst 0x45c69854  // ummla z20.s, z2.b, z6.b\n"
-      "ld1b { z4.b }, p0/Z, [x20, #2, MUL VL]\n"
+      "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n"
       ".inst 0x45c79857  // ummla z23.s, z2.b, z7.b\n"
       ".inst 0x45c6987a  // ummla z26.s, z3.b, z6.b\n"
-      "ld1b { z5.b }, p0/Z, [x20, #3, MUL VL]\n"
+      "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n"
       ".inst 0x45c7987d  // ummla z29.s, z3.b, z7.b\n"
-      "ld1b { z6.b }, p0/Z, [x20, #4, MUL VL]\n"
-      "ld1b { z7.b }, p0/Z, [x20, #5, MUL VL]\n"
+      "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n"
+      "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n"
       ".inst 0x45c49809  // ummla z9.s, z0.b, z4.b\n"
+      ".inst 0x45c5980c  // ummla z12.s, z0.b, z5.b\n"
+      "addvl x22, x22, #6\n"
       ".inst 0x45c4982f  // ummla z15.s, z1.b, z4.b\n"
+      ".inst 0x45c59832  // ummla z18.s, z1.b, z5.b\n"
       "add %x[Apanel], %x[Apanel], #0x40\n"
       ".inst 0x45c49855  // ummla z21.s, z2.b, z4.b\n"
+      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
       ".inst 0x45c4987b  // ummla z27.s, z3.b, z4.b\n"
-      "addvl x20, x20, #6\n"
-      ".inst 0x45c5980c  // ummla z12.s, z0.b, z5.b\n"
+      ".inst 0x45c5987e  // ummla z30.s, z3.b, z5.b\n"
       ".inst 0x45c6980a  // ummla z10.s, z0.b, z6.b\n"
       ".inst 0x45c7980d  // ummla z13.s, z0.b, z7.b\n"
-      ".inst 0x45c59832  // ummla z18.s, z1.b, z5.b\n"
       ".inst 0x45c69830  // ummla z16.s, z1.b, z6.b\n"
       ".inst 0x45c79833  // ummla z19.s, z1.b, z7.b\n"
-      ".inst 0x45c59858  // ummla z24.s, z2.b, z5.b\n"
-      ".inst 0x45c5987e  // ummla z30.s, z3.b, z5.b\n"
       ".inst 0x45c69856  // ummla z22.s, z2.b, z6.b\n"
       ".inst 0x45c79859  // ummla z25.s, z2.b, z7.b\n"
       ".inst 0x45c6987c  // ummla z28.s, z3.b, z6.b\n"
@@ -243,7 +243,7 @@ void sve_interleaved_u8u32_mmla_8x3VL(
       "uzp2 z14.d, z14.d, z17.d\n"
       "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
       "uzp1 z17.d, z15.d, z18.d\n"
-      "subs x22, x22, #0x1\n"
+      "subs x23, x23, #0x1\n"
       "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n"
       "uzp2 z15.d, z15.d, z18.d\n"
       "uzp1 z18.d, z16.d, z19.d\n"
@@ -285,7 +285,7 @@ void sve_interleaved_u8u32_mmla_8x3VL(
       "bne 1b\n"
       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
-      : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+      : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
     );
 }
 
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp
deleted file mode 100644
index ab225589e1..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2019-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_smallK_hybrid_fp32_mla_8x1VL(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
-class cls_sve_smallK_hybrid_fp32_mla_8x1VL
-{
-public:
-    typedef float operand_type;
-    typedef float result_type;
-
-    typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
-    /* Kernel blocking parameters */
-    static constexpr unsigned int out_height()
-    {
-        return 8;
-    }
-
-    static unsigned int out_width()
-    {
-        return get_vector_length<float>() * 1;
-    }
-
-    static constexpr unsigned int k_unroll()
-    {
-        return 1;
-    }
-
-    static constexpr bool supports_accumulate()
-    {
-        return false;
-    }
-
-    static constexpr bool supports_bias()
-    {
-        return true;
-    }
-
-    static constexpr bool supports_activation()
-    {
-        return true;
-    }
-
-    StdTransformsSVE<operand_type, result_type, 8, 1, 1> transforms = {};
-
-    // Default to the generic kernel
-    kern_type kernel=sve_smallK_hybrid_fp32_mla_8x1VL;
-
-    cls_sve_smallK_hybrid_fp32_mla_8x1VL(const CPUInfo *)
-    {
-
-    }
-};
-
-} // namespace arm_gemm
-
-#endif // ARM_COMPUTE_ENABLE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp
deleted file mode 100644
index c021539099..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp
+++ /dev/null
@@ -1,18807 +0,0 @@
-/*
- * Copyright (c) 2019-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_smallK_hybrid_fp32_mla_8x1VL(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
-    const long loops_count = iceildiv(N, (int)get_vector_length<float>()) - 1;
-    const long ldab = lda * sizeof(float);
-    const long ldcb = ldc * sizeof(float);
-    const long odd_depth  = (K % 4) ? (K % 4) : 4;
-    const long last_width = N - (loops_count * get_vector_length<float>());
-    float nullbias[64];
-    if (!bias) {
-        memset(nullbias, 0, (1 * get_vector_length<float>() * sizeof(float)));
-    }
-    float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
-    float maxval =   static_cast<float>(std::numeric_limits<float>::infinity());
-    const float * const minptr = &minval;
-    const float * const maxptr = &maxval;
-
-    switch(act.type)
-    {
-        default:
-        case Activation::Type::None:
-            break;
-        case Activation::Type::BoundedReLU:
-            maxval = static_cast<float>(act.param1);
-            /* fall through */
-        case Activation::Type::ReLU:
-            minval = 0.0f;
-            break;
-    }
-
-    for (int y0=0; y0<M; y0+=8) {
-        long loops = loops_count;
-        long oob_rows = std::max(8 - (M-y0), 0);
-        long temp = 0;
-        const float *b_ptr0 = B;
-        const float *biasptr = bias ? bias : nullbias;
-        const uint64_t biasinc = bias ? get_vector_length<float>() * 1*sizeof(float) : 0;
-        const float *a_ptr0 = A + (y0 * lda);
-
-        float *c_ptr0 = C + (y0 * ldc);
-
-        switch(K) {
-            case 1:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7]\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 2:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z28.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "mov z30.d, z24.d\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 3:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z28.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "mov z30.d, z24.d\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z28.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "mov z30.d, z24.d\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 4:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z25.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z26.d, z24.d\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z28.d, z24.d\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "mov z30.d, z24.d\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "mov z25.d, z24.d\n"
-                    "mov z26.d, z24.d\n"
-                    "mov z27.d, z24.d\n"
-                    "mov z28.d, z24.d\n"
-                    "mov z29.d, z24.d\n"
-                    "mov z30.d, z24.d\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 5:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 6:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z27.d, z24.d\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 7:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z27.d, z24.d\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 8:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 9:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 10:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 11:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 12:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 13:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 14:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 15:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 16:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 17:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 18:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 19:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 20:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 21:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 22:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 23:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            default:
-            case 24:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.s\n"
-                    "whilelt p6.s, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z31.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p7/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "mov z25.d, z24.d\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "ld1w z24.s, p0/z, [%[biasptr]]\n"
-                    "add %[biasptr], %[biasptr], %[biasinc]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1]\n"
-                    "mov z25.d, z24.d\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2]\n"
-                    "mov z26.d, z24.d\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3]\n"
-                    "mov z27.d, z24.d\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4]\n"
-                    "mov z28.d, z24.d\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5]\n"
-                    "mov z29.d, z24.d\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6]\n"
-                    "mov z30.d, z24.d\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7]\n"
-                    "mov z31.d, z24.d\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x10]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x10]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x10]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x10]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x20]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x20]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x20]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x20]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "ld1w z16.s, p7/z, [%[b_ptr0]]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "ld1w z17.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "ld1w z18.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x30]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x30]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x30]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x30]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "ld1w z19.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "ld1w z20.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "ld1w z21.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "ld1w z22.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p7/z, [a_ptr4, #0x40]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p7/z, [a_ptr5, #0x40]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p7/z, [a_ptr6, #0x40]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "ld1w z23.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "fmla z24.s, z16.s, z0.s[0]\n"
-                    "ld1rqw z7.s, p7/z, [a_ptr7, #0x40]\n"
-                    "fmla z25.s, z16.s, z1.s[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "fmla z26.s, z16.s, z2.s[0]\n"
-                    "fmla z27.s, z16.s, z3.s[0]\n"
-                    "fmla z28.s, z16.s, z4.s[0]\n"
-                    "fmla z29.s, z16.s, z5.s[0]\n"
-                    "fmla z30.s, z16.s, z6.s[0]\n"
-                    "fmla z31.s, z16.s, z7.s[0]\n"
-                    "fmla z24.s, z17.s, z0.s[1]\n"
-                    "fmla z25.s, z17.s, z1.s[1]\n"
-                    "fmla z26.s, z17.s, z2.s[1]\n"
-                    "fmla z27.s, z17.s, z3.s[1]\n"
-                    "fmla z28.s, z17.s, z4.s[1]\n"
-                    "fmla z29.s, z17.s, z5.s[1]\n"
-                    "fmla z30.s, z17.s, z6.s[1]\n"
-                    "fmla z31.s, z17.s, z7.s[1]\n"
-                    "fmla z24.s, z18.s, z0.s[2]\n"
-                    "fmla z25.s, z18.s, z1.s[2]\n"
-                    "fmla z26.s, z18.s, z2.s[2]\n"
-                    "fmla z27.s, z18.s, z3.s[2]\n"
-                    "fmla z28.s, z18.s, z4.s[2]\n"
-                    "fmla z29.s, z18.s, z5.s[2]\n"
-                    "fmla z30.s, z18.s, z6.s[2]\n"
-                    "fmla z31.s, z18.s, z7.s[2]\n"
-                    "fmla z24.s, z19.s, z0.s[3]\n"
-                    "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
-                    "fmla z25.s, z19.s, z1.s[3]\n"
-                    "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
-                    "fmla z26.s, z19.s, z2.s[3]\n"
-                    "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
-                    "fmla z27.s, z19.s, z3.s[3]\n"
-                    "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
-                    "fmla z28.s, z19.s, z4.s[3]\n"
-                    "ld1rqw z4.s, p6/z, [a_ptr4, #0x50]\n"
-                    "fmla z29.s, z19.s, z5.s[3]\n"
-                    "ld1rqw z5.s, p6/z, [a_ptr5, #0x50]\n"
-                    "fmla z30.s, z19.s, z6.s[3]\n"
-                    "ld1rqw z6.s, p6/z, [a_ptr6, #0x50]\n"
-                    "fmla z31.s, z19.s, z7.s[3]\n"
-                    "ld1rqw z7.s, p6/z, [a_ptr7, #0x50]\n"
-                    "fmla z24.s, z20.s, z0.s[0]\n"
-                    "fmla z25.s, z20.s, z1.s[0]\n"
-                    "fmla z26.s, z20.s, z2.s[0]\n"
-                    "fmla z27.s, z20.s, z3.s[0]\n"
-                    "fmla z28.s, z20.s, z4.s[0]\n"
-                    "fmla z29.s, z20.s, z5.s[0]\n"
-                    "fmla z30.s, z20.s, z6.s[0]\n"
-                    "fmla z31.s, z20.s, z7.s[0]\n"
-                    "fmla z24.s, z21.s, z0.s[1]\n"
-                    "fmla z25.s, z21.s, z1.s[1]\n"
-                    "fmla z26.s, z21.s, z2.s[1]\n"
-                    "fmla z27.s, z21.s, z3.s[1]\n"
-                    "fmla z28.s, z21.s, z4.s[1]\n"
-                    "fmla z29.s, z21.s, z5.s[1]\n"
-                    "fmla z30.s, z21.s, z6.s[1]\n"
-                    "fmla z31.s, z21.s, z7.s[1]\n"
-                    "fmla z24.s, z22.s, z0.s[2]\n"
-                    "fmla z25.s, z22.s, z1.s[2]\n"
-                    "fmla z26.s, z22.s, z2.s[2]\n"
-                    "fmla z27.s, z22.s, z3.s[2]\n"
-                    "fmla z28.s, z22.s, z4.s[2]\n"
-                    "fmla z29.s, z22.s, z5.s[2]\n"
-                    "fmla z30.s, z22.s, z6.s[2]\n"
-                    "fmla z31.s, z22.s, z7.s[2]\n"
-                    "fmla z24.s, z23.s, z0.s[3]\n"
-                    "fmla z25.s, z23.s, z1.s[3]\n"
-                    "fmla z26.s, z23.s, z2.s[3]\n"
-                    "fmla z27.s, z23.s, z3.s[3]\n"
-                    "fmla z28.s, z23.s, z4.s[3]\n"
-                    "fmla z29.s, z23.s, z5.s[3]\n"
-                    "fmla z30.s, z23.s, z6.s[3]\n"
-                    "fmla z31.s, z23.s, z7.s[3]\n"
-                    "5:\n"
-                    "ld1rw z22.s, p7/z, [%[minptr]]\n"
-                    "ld1rw z23.s, p7/z, [%[maxptr]]\n"
-                    "fmax z24.s, p7/m, z24.s, z22.s\n"
-                    "fmax z25.s, p7/m, z25.s, z22.s\n"
-                    "fmax z26.s, p7/m, z26.s, z22.s\n"
-                    "fmax z27.s, p7/m, z27.s, z22.s\n"
-                    "fmin z24.s, p7/m, z24.s, z23.s\n"
-                    "fmin z25.s, p7/m, z25.s, z23.s\n"
-                    "fmin z26.s, p7/m, z26.s, z23.s\n"
-                    "fmin z27.s, p7/m, z27.s, z23.s\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "fmax z28.s, p7/m, z28.s, z22.s\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "fmax z29.s, p7/m, z29.s, z22.s\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "fmax z30.s, p7/m, z30.s, z22.s\n"
-                    "fmin z28.s, p7/m, z28.s, z23.s\n"
-                    "fmax z31.s, p7/m, z31.s, z22.s\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "fmin z29.s, p7/m, z29.s, z23.s\n"
-                    "fmin z30.s, p7/m, z30.s, z23.s\n"
-                    "fmin z31.s, p7/m, z31.s, z23.s\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [biasptr] "+r" (biasptr)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-        }
-    }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp
deleted file mode 100644
index e735567e95..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2019-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <cstdint>
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_smallK_hybrid_s8s32_dot_8x1VL(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-
-class cls_sve_smallK_hybrid_s8s32_dot_8x1VL
-{
-public:
-    typedef int8_t operand_type;
-    typedef int32_t result_type;
-
-    typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-
-    /* Kernel blocking parameters */
-    static constexpr unsigned int out_height()
-    {
-        return 8;
-    }
-
-    static unsigned int out_width()
-    {
-        return get_vector_length<int32_t>() * 1;
-    }
-
-    static constexpr unsigned int k_unroll()
-    {
-        return 4;
-    }
-
-    static constexpr bool supports_accumulate()
-    {
-        return false;
-    }
-
-    static constexpr bool supports_bias()
-    {
-        return false;
-    }
-
-    static constexpr bool supports_activation()
-    {
-        return false;
-    }
-
-    StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
-
-    // Default to the generic kernel
-    kern_type kernel=sve_smallK_hybrid_s8s32_dot_8x1VL;
-
-    cls_sve_smallK_hybrid_s8s32_dot_8x1VL(const CPUInfo *)
-    {
-
-    }
-};
-
-} // namespace arm_gemm
-
-#endif // ARM_COMPUTE_ENABLE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp
deleted file mode 100644
index 489b381624..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp
+++ /dev/null
@@ -1,8747 +0,0 @@
-/*
- * Copyright (c) 2019-2021 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_smallK_hybrid_s8s32_dot_8x1VL(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
-    const long loops_count = iceildiv(N, (int)get_vector_length<int32_t>()) - 1;
-    const long ldab = lda * sizeof(int8_t);
-    const long ldcb = ldc * sizeof(int32_t);
-    const long odd_depth  = (K % 16) ? (K % 16) : 16;
-    const long last_width = N - (loops_count * get_vector_length<int32_t>());
-    const long odds_count = K % 4;
-    K = (K + 3) / 4;
-
-    for (int y0=0; y0<M; y0+=8) {
-        long loops = loops_count;
-        long oob_rows = std::max(8 - (M-y0), 0);
-        long odds = odds_count;
-        long temp = 0;
-        const int8_t *b_ptr0 = B;
-        const int8_t *a_ptr0 = A + (y0 * lda);
-
-        int32_t *c_ptr0 = C + (y0 * ldc);
-
-        switch(K) {
-            case 1:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 2:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 3:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 4:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 5:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 6:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 7:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 8:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 9:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 10:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 11:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 12:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 13:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 14:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 15:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            default:
-            case 16:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "sdot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "sdot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "sdot z26.s, z16.b, z2.b[0]\n"
-                    "sdot z27.s, z16.b, z3.b[0]\n"
-                    "sdot z28.s, z16.b, z4.b[0]\n"
-                    "sdot z29.s, z16.b, z5.b[0]\n"
-                    "sdot z30.s, z16.b, z6.b[0]\n"
-                    "sdot z31.s, z16.b, z7.b[0]\n"
-                    "sdot z24.s, z17.b, z0.b[1]\n"
-                    "sdot z25.s, z17.b, z1.b[1]\n"
-                    "sdot z26.s, z17.b, z2.b[1]\n"
-                    "sdot z27.s, z17.b, z3.b[1]\n"
-                    "sdot z28.s, z17.b, z4.b[1]\n"
-                    "sdot z29.s, z17.b, z5.b[1]\n"
-                    "sdot z30.s, z17.b, z6.b[1]\n"
-                    "sdot z31.s, z17.b, z7.b[1]\n"
-                    "sdot z24.s, z18.b, z0.b[2]\n"
-                    "sdot z25.s, z18.b, z1.b[2]\n"
-                    "sdot z26.s, z18.b, z2.b[2]\n"
-                    "sdot z27.s, z18.b, z3.b[2]\n"
-                    "sdot z28.s, z18.b, z4.b[2]\n"
-                    "sdot z29.s, z18.b, z5.b[2]\n"
-                    "sdot z30.s, z18.b, z6.b[2]\n"
-                    "sdot z31.s, z18.b, z7.b[2]\n"
-                    "sdot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "sdot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "sdot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "sdot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "sdot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "sdot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "sdot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "sdot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "sdot z24.s, z20.b, z0.b[0]\n"
-                    "sdot z25.s, z20.b, z1.b[0]\n"
-                    "sdot z26.s, z20.b, z2.b[0]\n"
-                    "sdot z27.s, z20.b, z3.b[0]\n"
-                    "sdot z28.s, z20.b, z4.b[0]\n"
-                    "sdot z29.s, z20.b, z5.b[0]\n"
-                    "sdot z30.s, z20.b, z6.b[0]\n"
-                    "sdot z31.s, z20.b, z7.b[0]\n"
-                    "sdot z24.s, z21.b, z0.b[1]\n"
-                    "sdot z25.s, z21.b, z1.b[1]\n"
-                    "sdot z26.s, z21.b, z2.b[1]\n"
-                    "sdot z27.s, z21.b, z3.b[1]\n"
-                    "sdot z28.s, z21.b, z4.b[1]\n"
-                    "sdot z29.s, z21.b, z5.b[1]\n"
-                    "sdot z30.s, z21.b, z6.b[1]\n"
-                    "sdot z31.s, z21.b, z7.b[1]\n"
-                    "sdot z24.s, z22.b, z0.b[2]\n"
-                    "sdot z25.s, z22.b, z1.b[2]\n"
-                    "sdot z26.s, z22.b, z2.b[2]\n"
-                    "sdot z27.s, z22.b, z3.b[2]\n"
-                    "sdot z28.s, z22.b, z4.b[2]\n"
-                    "sdot z29.s, z22.b, z5.b[2]\n"
-                    "sdot z30.s, z22.b, z6.b[2]\n"
-                    "sdot z31.s, z22.b, z7.b[2]\n"
-                    "sdot z24.s, z23.b, z0.b[3]\n"
-                    "sdot z25.s, z23.b, z1.b[3]\n"
-                    "sdot z26.s, z23.b, z2.b[3]\n"
-                    "sdot z27.s, z23.b, z3.b[3]\n"
-                    "sdot z28.s, z23.b, z4.b[3]\n"
-                    "sdot z29.s, z23.b, z5.b[3]\n"
-                    "sdot z30.s, z23.b, z6.b[3]\n"
-                    "sdot z31.s, z23.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-        }
-    }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp
deleted file mode 100644
index 25dd10019d..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2019-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <cstdint>
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_smallK_hybrid_u8u32_dot_8x1VL(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-
-class cls_sve_smallK_hybrid_u8u32_dot_8x1VL
-{
-public:
-    typedef uint8_t operand_type;
-    typedef uint32_t result_type;
-
-    typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-
-    /* Kernel blocking parameters */
-    static constexpr unsigned int out_height()
-    {
-        return 8;
-    }
-
-    static unsigned int out_width()
-    {
-        return get_vector_length<uint32_t>() * 1;
-    }
-
-    static constexpr unsigned int k_unroll()
-    {
-        return 4;
-    }
-
-    static constexpr bool supports_accumulate()
-    {
-        return false;
-    }
-
-    static constexpr bool supports_bias()
-    {
-        return false;
-    }
-
-    static constexpr bool supports_activation()
-    {
-        return false;
-    }
-
-    StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
-
-    // Default to the generic kernel
-    kern_type kernel=sve_smallK_hybrid_u8u32_dot_8x1VL;
-
-    cls_sve_smallK_hybrid_u8u32_dot_8x1VL(const CPUInfo *)
-    {
-
-    }
-};
-
-} // namespace arm_gemm
-
-#endif // ARM_COMPUTE_ENABLE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp
deleted file mode 100644
index 8ab83e670e..0000000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp
+++ /dev/null
@@ -1,8747 +0,0 @@
-/*
- * Copyright (c) 2019-2021 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_smallK_hybrid_u8u32_dot_8x1VL(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
-    const long loops_count = iceildiv(N, (int)get_vector_length<uint32_t>()) - 1;
-    const long ldab = lda * sizeof(uint8_t);
-    const long ldcb = ldc * sizeof(uint32_t);
-    const long odd_depth  = (K % 16) ? (K % 16) : 16;
-    const long last_width = N - (loops_count * get_vector_length<uint32_t>());
-    const long odds_count = K % 4;
-    K = (K + 3) / 4;
-
-    for (int y0=0; y0<M; y0+=8) {
-        long loops = loops_count;
-        long oob_rows = std::max(8 - (M-y0), 0);
-        long odds = odds_count;
-        long temp = 0;
-        const uint8_t *b_ptr0 = B;
-        const uint8_t *a_ptr0 = A + (y0 * lda);
-
-        uint32_t *c_ptr0 = C + (y0 * ldc);
-
-        switch(K) {
-            case 1:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 2:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 3:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 4:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "mov z27.s, #0\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "mov z25.s, #0\n"
-                    "mov z26.s, #0\n"
-                    "mov z27.s, #0\n"
-                    "mov z28.s, #0\n"
-                    "mov z29.s, #0\n"
-                    "mov z30.s, #0\n"
-                    "mov z31.s, #0\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 5:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 6:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 7:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 8:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 9:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #1\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 10:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #2\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 11:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #3\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 12:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #4\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 13:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #5\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 14:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #6\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            case 15:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #7\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-            default:
-            case 16:
-                __asm __volatile (
-                    "a_ptr1 .req X0\n"
-                    "a_ptr2 .req X1\n"
-                    "a_ptr3 .req X2\n"
-                    "a_ptr4 .req X3\n"
-                    "a_ptr5 .req X4\n"
-                    "a_ptr6 .req X5\n"
-                    "a_ptr7 .req X6\n"
-                    "c_ptr1 .req X7\n"
-                    "c_ptr2 .req X8\n"
-                    "c_ptr3 .req X9\n"
-                    "c_ptr4 .req X10\n"
-                    "c_ptr5 .req X11\n"
-                    "c_ptr6 .req X12\n"
-                    "c_ptr7 .req X13\n"
-                    "add a_ptr1, %[a_ptr0], %[lda]\n"
-                    "add c_ptr1, %[c_ptr0], %[ldc]\n"
-                    "add a_ptr2, a_ptr1, %[lda]\n"
-                    "add c_ptr2, c_ptr1, %[ldc]\n"
-                    "add a_ptr3, a_ptr2, %[lda]\n"
-                    "add c_ptr3, c_ptr2, %[ldc]\n"
-                    "add a_ptr4, a_ptr3, %[lda]\n"
-                    "add c_ptr4, c_ptr3, %[ldc]\n"
-                    "add a_ptr5, a_ptr4, %[lda]\n"
-                    "add c_ptr5, c_ptr4, %[ldc]\n"
-                    "add a_ptr6, a_ptr5, %[lda]\n"
-                    "add c_ptr6, c_ptr5, %[ldc]\n"
-                    "add a_ptr7, a_ptr6, %[lda]\n"
-                    "add c_ptr7, c_ptr6, %[ldc]\n"
-                    "cbz %[oob_rows], 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr7, %[c_ptr0], #0x0\n"
-                    "add a_ptr7, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr6, %[c_ptr0], #0x0\n"
-                    "add a_ptr6, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr5, %[c_ptr0], #0x0\n"
-                    "add a_ptr5, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr4, %[c_ptr0], #0x0\n"
-                    "add a_ptr4, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr3, %[c_ptr0], #0x0\n"
-                    "add a_ptr3, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr2, %[c_ptr0], #0x0\n"
-                    "add a_ptr2, %[a_ptr0], #0x0\n"
-                    "b.eq 1f\n"
-                    "subs %[oob_rows], %[oob_rows], #0x1\n"
-                    "add c_ptr1, %[c_ptr0], #0x0\n"
-                    "add a_ptr1, %[a_ptr0], #0x0\n"
-                    "1:\n"
-                    "ptrue p7.b\n"
-                    "whilelt p6.b, %[temp], %[odd_depth]\n"
-                    "whilelt p0.s, %[temp], %[last_width]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "cbz %[loops], 2f\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b.eq 3f\n"
-                    "4:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "subs %[loops], %[loops], #0x1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b.ne 4b\n"
-                    "3:\n"
-                    "st1w z24.s, p7, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "mov z24.s, #0\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "st1w z25.s, p7, [c_ptr1]\n"
-                    "addvl c_ptr1, c_ptr1, #1\n"
-                    "mov z25.s, #0\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "st1w z26.s, p7, [c_ptr2]\n"
-                    "addvl c_ptr2, c_ptr2, #1\n"
-                    "mov z26.s, #0\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "st1w z27.s, p7, [c_ptr3]\n"
-                    "addvl c_ptr3, c_ptr3, #1\n"
-                    "mov z27.s, #0\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "st1w z28.s, p7, [c_ptr4]\n"
-                    "addvl c_ptr4, c_ptr4, #1\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "st1w z29.s, p7, [c_ptr5]\n"
-                    "addvl c_ptr5, c_ptr5, #1\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "st1w z30.s, p7, [c_ptr6]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "st1w z31.s, p7, [c_ptr7]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "addvl c_ptr6, c_ptr6, #1\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "addvl c_ptr7, c_ptr7, #1\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "b 5f\n"
-                    "2:\n"
-                    "mov z24.s, #0\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
-                    "mov z25.s, #0\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1]\n"
-                    "mov z26.s, #0\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2]\n"
-                    "mov z27.s, #0\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3]\n"
-                    "mov z28.s, #0\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4]\n"
-                    "mov z29.s, #0\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5]\n"
-                    "mov z30.s, #0\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6]\n"
-                    "mov z31.s, #0\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
-                    "udot z24.s, z16.b, z0.b[0]\n"
-                    "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
-                    "udot z25.s, z16.b, z1.b[0]\n"
-                    "addvl %[b_ptr0], %[b_ptr0], #8\n"
-                    "udot z26.s, z16.b, z2.b[0]\n"
-                    "udot z27.s, z16.b, z3.b[0]\n"
-                    "udot z28.s, z16.b, z4.b[0]\n"
-                    "udot z29.s, z16.b, z5.b[0]\n"
-                    "udot z30.s, z16.b, z6.b[0]\n"
-                    "udot z31.s, z16.b, z7.b[0]\n"
-                    "udot z24.s, z17.b, z0.b[1]\n"
-                    "udot z25.s, z17.b, z1.b[1]\n"
-                    "udot z26.s, z17.b, z2.b[1]\n"
-                    "udot z27.s, z17.b, z3.b[1]\n"
-                    "udot z28.s, z17.b, z4.b[1]\n"
-                    "udot z29.s, z17.b, z5.b[1]\n"
-                    "udot z30.s, z17.b, z6.b[1]\n"
-                    "udot z31.s, z17.b, z7.b[1]\n"
-                    "udot z24.s, z18.b, z0.b[2]\n"
-                    "udot z25.s, z18.b, z1.b[2]\n"
-                    "udot z26.s, z18.b, z2.b[2]\n"
-                    "udot z27.s, z18.b, z3.b[2]\n"
-                    "udot z28.s, z18.b, z4.b[2]\n"
-                    "udot z29.s, z18.b, z5.b[2]\n"
-                    "udot z30.s, z18.b, z6.b[2]\n"
-                    "udot z31.s, z18.b, z7.b[2]\n"
-                    "udot z24.s, z19.b, z0.b[3]\n"
-                    "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
-                    "udot z25.s, z19.b, z1.b[3]\n"
-                    "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
-                    "udot z26.s, z19.b, z2.b[3]\n"
-                    "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
-                    "udot z27.s, z19.b, z3.b[3]\n"
-                    "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
-                    "udot z28.s, z19.b, z4.b[3]\n"
-                    "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
-                    "udot z29.s, z19.b, z5.b[3]\n"
-                    "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
-                    "udot z30.s, z19.b, z6.b[3]\n"
-                    "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
-                    "udot z31.s, z19.b, z7.b[3]\n"
-                    "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
-                    "udot z24.s, z20.b, z0.b[0]\n"
-                    "udot z25.s, z20.b, z1.b[0]\n"
-                    "udot z26.s, z20.b, z2.b[0]\n"
-                    "udot z27.s, z20.b, z3.b[0]\n"
-                    "udot z28.s, z20.b, z4.b[0]\n"
-                    "udot z29.s, z20.b, z5.b[0]\n"
-                    "udot z30.s, z20.b, z6.b[0]\n"
-                    "udot z31.s, z20.b, z7.b[0]\n"
-                    "udot z24.s, z21.b, z0.b[1]\n"
-                    "udot z25.s, z21.b, z1.b[1]\n"
-                    "udot z26.s, z21.b, z2.b[1]\n"
-                    "udot z27.s, z21.b, z3.b[1]\n"
-                    "udot z28.s, z21.b, z4.b[1]\n"
-                    "udot z29.s, z21.b, z5.b[1]\n"
-                    "udot z30.s, z21.b, z6.b[1]\n"
-                    "udot z31.s, z21.b, z7.b[1]\n"
-                    "udot z24.s, z22.b, z0.b[2]\n"
-                    "udot z25.s, z22.b, z1.b[2]\n"
-                    "udot z26.s, z22.b, z2.b[2]\n"
-                    "udot z27.s, z22.b, z3.b[2]\n"
-                    "udot z28.s, z22.b, z4.b[2]\n"
-                    "udot z29.s, z22.b, z5.b[2]\n"
-                    "udot z30.s, z22.b, z6.b[2]\n"
-                    "udot z31.s, z22.b, z7.b[2]\n"
-                    "udot z24.s, z23.b, z0.b[3]\n"
-                    "udot z25.s, z23.b, z1.b[3]\n"
-                    "udot z26.s, z23.b, z2.b[3]\n"
-                    "udot z27.s, z23.b, z3.b[3]\n"
-                    "udot z28.s, z23.b, z4.b[3]\n"
-                    "udot z29.s, z23.b, z5.b[3]\n"
-                    "udot z30.s, z23.b, z6.b[3]\n"
-                    "udot z31.s, z23.b, z7.b[3]\n"
-                    "5:\n"
-                    "st1w z24.s, p0, [%[c_ptr0]]\n"
-                    "addvl %[c_ptr0], %[c_ptr0], #1\n"
-                    "st1w z25.s, p0, [c_ptr1]\n"
-                    "st1w z26.s, p0, [c_ptr2]\n"
-                    "st1w z27.s, p0, [c_ptr3]\n"
-                    "st1w z28.s, p0, [c_ptr4]\n"
-                    "st1w z29.s, p0, [c_ptr5]\n"
-                    "st1w z30.s, p0, [c_ptr6]\n"
-                    "st1w z31.s, p0, [c_ptr7]\n"
-                    ".unreq a_ptr1\n"
-                    ".unreq a_ptr2\n"
-                    ".unreq a_ptr3\n"
-                    ".unreq a_ptr4\n"
-                    ".unreq a_ptr5\n"
-                    ".unreq a_ptr6\n"
-                    ".unreq a_ptr7\n"
-                    ".unreq c_ptr1\n"
-                    ".unreq c_ptr2\n"
-                    ".unreq c_ptr3\n"
-                    ".unreq c_ptr4\n"
-                    ".unreq c_ptr5\n"
-                    ".unreq c_ptr6\n"
-                    ".unreq c_ptr7\n"
-                    : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [temp] "+r" (temp), [odds] "+r" (odds)
-                    : [lda] "r" (ldab), [ldc] "r" (ldcb), [odd_depth] "r" (odd_depth), [last_width] "r" (last_width)
-                    : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
-                );
-                break;
-        }
-    }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE